merge start.sh and start.py

Author: naderzare

scripts/create_binary.sh

@@ -32,15 +32,6 @@ rm -rf binary/start.build
 rm -rf binary/start.dist
 rm -rf binary/start.onefile-build
 
-# copy start.sh to binary directory
-
-cp ../start.sh binary/start.sh
-
-# change start.sh to run binary instead of python and to use separate rpc server
-
-sed -i 's/run_bin=false/run_bin=true/g' binary/start.sh
-sed -i 's/separate_rpc_server=false/separate_rpc_server=true/g' binary/start.sh
-
 # copy start to binary directory
 
 cp start binary/start

start.py

@@ -4,32 +4,40 @@
 import threading
 import logging
 import argparse
-import check_requirements
 from utils.logger_utils import setup_logger
 import datetime
 import multiprocessing
 from server import main
 import random
+import select
+
 
 # Set up logging
 log_dir = None
 start_team_logger = None
 
 def run_server_script(args):
+    rpc_port = args.rpc_port
+    if args.use_random_port or args.use_different_port:
+        import socket
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.bind(('localhost', 0))
+            rpc_port = str(s.getsockname()[1])
+        start_team_logger.debug(f"Using random port: {rpc_port}")
     # Define a wrapper function to pass the arguments to the main function
     def server_main():
         import sys
-        sys.argv = ['server.py', '--rpc-port', str(args.rpc_port), '--log-dir', log_dir]
+        sys.argv = ['server.py', '--rpc-port', str(rpc_port), '--log-dir', log_dir]
         if args.disable_log_file:
             sys.argv += ['--disable-log-file']
         main()
 
     # Start the main function as a new process
     process = multiprocessing.Process(target=server_main)
     process.start()
-    return process
+    return process, rpc_port
 
-def run_start_script(args):
+def run_start_script(args, rpc_port):
     # Start the start.sh script in its own directory as a new process group
     arguments = ['bash']
     if args.player or args.coach or args.goalie:
@@ -38,7 +46,7 @@ def run_start_script(args):
         arguments += ['start.sh' if not args.debug else 'start-debug.sh']
 
     arguments += ['-t', args.team_name, 
-         '--rpc-port', args.rpc_port, '--rpc-type', 'grpc', 
+         '--rpc-port', rpc_port, '--rpc-type', 'grpc', 
          '-p', args.server_port, '-h', args.server_host]
 
     process = subprocess.Popen(
@@ -56,32 +64,39 @@ def stream_output_to_file(process, name, args):
     # Stream output from the process and log it to a file with the given name
     f = None if args.disable_log_file else open(f"{log_dir}/{name}.log", "a")
     while not stop_thread.is_set():
-        output = process.stdout.readline()
-        if output:
-            if f:
-                f.write(output.decode())
-                f.flush()
+        if select.select([process.stdout], [], [], 1)[0]:
+            output = process.stdout.readline()
+            if output:
+                if f:
+                    f.write(output.decode())
+                    f.flush()
+                else:
+                    print(output.decode())
             else:
-                print(output.decode())
-        else:
-            break
+                break
     process.stdout.close()
+    if f:
+        f.close()
 
-def kill_process_group(process):
-    try:
-        start_team_logger.debug(f"Killing process group with PID: {process.pid}")
-        os.killpg(os.getpgid(process.pid), signal.SIGTERM)  # Send SIGTERM to the process group
-    except ProcessLookupError:
-        pass  # The process might have already exited
+def kill_process_group(processes):
+    for process in processes:
+        try:
+            start_team_logger.debug(f"Killing process group with PID: {process.pid}")
+            os.killpg(os.getpgid(process.pid), signal.SIGTERM)  # Send SIGTERM to the process group
+        except ProcessLookupError:
+            pass  # The process might have already exited
 
-def kill_rpc_server_process(process):
-    try:
-        start_team_logger.debug(f"Killing process with PID: {process.pid}")
-        os.kill(process.pid, signal.SIGTERM)  # Send SIGTERM to the specific process
-    except ProcessLookupError:
-        pass  # The process might have already exited
+def kill_rpc_server_process(processes):
+    for process in processes:
+        try:
+            start_team_logger.debug(f"Killing process with PID: {process.pid}")
+            os.kill(process.pid, signal.SIGTERM)  # Send SIGTERM to the specific process
+        except ProcessLookupError:
+            pass  # The process might have already exited
+
+def check_args(args):
+    pass
 
-    
 if __name__ == "__main__":
     # Set up argument parsing
     parser = argparse.ArgumentParser(description='Run server and team scripts.')
@@ -90,6 +105,7 @@ def kill_rpc_server_process(process):
     parser.add_argument('-d', '--debug', required=False, help='Enable debug mode', default=False, action='store_true')
     parser.add_argument('--use-random-port', required=False, help='Use a random port for the server', default=False, action='store_true')
     parser.add_argument('--use-random-name', required=False, help='Use a random team name', default=False, action='store_true')
+    parser.add_argument('--use-different-port', required=False, help='Use a different port for the rpc server', default=False, action='store_true')
     parser.add_argument('--server-host', required=False, help='The host of the server', default='localhost')
     parser.add_argument('--server-port', required=False, help='The port of the server', default='6000')
     parser.add_argument('--close-server', required=False, help='Close the server', default=False, action='store_true')
@@ -110,47 +126,70 @@ def kill_rpc_server_process(process):
     start_team_logger.debug(f"Arguments: {args=}")
 
     try:
-        if args.use_random_port:
-            import socket
-            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-                s.bind(('localhost', 0))
-                args.rpc_port = str(s.getsockname()[1])
-            start_team_logger.debug(f"Using random port: {args.rpc_port}")
-                
         if args.use_random_name:
             import random
             import string
             args.team_name = ''.join(random.choices(string.ascii_uppercase + string.digits, k=5))
 
         # Run the server.py script first
-        server_process = run_server_script(args)
-        start_team_logger.debug(f"Started server.py process with PID: {server_process.pid}")
-        
-        # Run the start.sh script after server.py with the given arguments
-        start_process = run_start_script(args)
-        start_team_logger.debug(f"Started start.sh process with PID: {start_process.pid} with team name {args=}")
+        all_server_processes = []
+        all_start_processes = []
+        if args.use_different_port:
+            for i in range(13):
+                server_process, rpc_port = run_server_script(args)
+                all_server_processes.append(server_process)
+                start_team_logger.debug(f"Started server.py process with PID: {all_server_processes[-1].pid}")
+                args.goalie, args.coach, args.player = False, False, False
+                args.player = False
+                if i == 0:
+                    args.goalie = True
+                elif i == 12:
+                    args.coach = True
+                else:
+                    args.player = True
+                
+                # Run the start.sh script after server.py with the given arguments
+                all_start_processes.append(run_start_script(args, rpc_port))
+                start_team_logger.debug(f"Started start.sh process with PID: {all_start_processes[-1].pid} with team name {args=} and rpc port {rpc_port}")
+        else:
+            server_process, rpc_port = run_server_script(args)
+            all_server_processes.append(server_process)
+            start_team_logger.debug(f"Started server.py process with PID: {all_server_processes[-1].pid}")
+            
+            # Run the start.sh script after server.py with the given arguments
+            all_start_processes.append(run_start_script(args, rpc_port))
+            start_team_logger.debug(f"Started start.sh process with PID: {all_start_processes[-1].pid} with team name {args=} and rpc port {rpc_port}")
 
         # Monitor both processes and log their outputs
         start_team_logger.debug("Monitoring processes...")
 
-        start_thread = threading.Thread(target=stream_output_to_file, args=(start_process, 'proxy', args))
-        start_thread.start()
+        start_threads: list[threading.Thread] = []
+        for i, start_process in enumerate(all_start_processes):
+            start_threads.append(threading.Thread(target=stream_output_to_file, args=(start_process, 'proxy' if len(all_start_processes) == 1 else f'porxy_{i}', args)))
+            start_threads[-1].start()
 
         def signal_handler(sig, frame):
             start_team_logger.info('Received signal to terminate. Cleaning up...')
-            kill_rpc_server_process(server_process)
-            server_process.join()
-            
-            start_team_logger.debug(f"server.py ended with PID: {server_process.pid}")
+            kill_rpc_server_process(all_server_processes)
+            for server_process in all_server_processes:
+                server_process.join()
+                start_team_logger.debug(f"server.py ended with PID: {server_process.pid}")
+                
+            start_team_logger.debug("all_server_processes has finished.")
 
-            kill_process_group(start_process)
-            start_process.wait()
+            kill_process_group(all_start_processes)
+            for start_process in all_start_processes:
+                start_process.wait()
+                start_team_logger.debug(f"start.sh ended with PID: {start_process.pid}")
 
-            start_team_logger.debug(f"start.sh ended with PID: {start_process.pid}")
+            start_team_logger.debug("all_start_processes has finished.")
 
             stop_thread.set()
 
-            start_thread.join()
+            start_team_logger.debug("Waiting for start_thread to finish...")
+            for start_thread in start_threads:
+                start_thread.join()
+            
             start_team_logger.debug("start_thread has finished.")
 
             start_team_logger.info('All processes have been killed.')
@@ -160,26 +199,28 @@ def signal_handler(sig, frame):
         signal.signal(signal.SIGINT, signal_handler)
 
         # Wait for both threads to finish
-        start_thread.join()
+        for start_thread in start_threads:
+            start_thread.join()
         start_team_logger.debug("agents has been exited.")
 
         if args.close_server:
             start_team_logger.debug("Closing rpc server ...")
-            kill_rpc_server_process(server_process)
+            kill_rpc_server_process(all_server_processes)
             start_team_logger.debug("rpc Server has been closed.")
 
         # wait for server process to finish
-        server_process.join()
+        for server_process in all_server_processes:
+            server_process.join()
         start_team_logger.debug("server.py has finished.")
 
         start_team_logger.info("Both processes have exited.")
     except KeyboardInterrupt:
         start_team_logger.debug("Interrupted! Killing all processes.")
-        kill_rpc_server_process(server_process)
-        kill_process_group(start_process)
+        kill_rpc_server_process(all_server_processes)
+        kill_process_group(all_start_processes)
 
     finally:
         # Ensure all processes are killed on exit
         start_team_logger.debug("Final Cleaning up...")
-        kill_rpc_server_process(server_process)
-        kill_process_group(start_process)
+        kill_rpc_server_process(all_server_processes)
+        kill_process_group(all_start_processes)

start.sh

@@ -1,103 +1,27 @@
 #!/bin/bash
 
 python_env=.venv/bin/activate
+source $python_env
 
-server_host=127.0.0.1
-server_port=6000
-disable_log_file=false
-run_bin=false
-separate_rpc_server=false
+# Set the options, all the options are passed as environment variables
+options=$@
 
-while [ $# -gt 0 ]
-do
-  case $1 in
-    --server-host)
-      server_host=$2
-      shift
-      ;;
-    --server-port)
-      server_port=$2
-      shift
-      ;;
-    --disable_log_file)
-      disable_log_file=true
-      ;;
-    --run-bin)
-      run_bin=true
-      ;;
-    --separate-rpc-server)
-      separate_rpc_server=true
-      ;;
-    *)
-      echo 1>&2
-      echo "invalid option \"${1}\"." 1>&2
-      echo 1>&2
-      usage
-      exit 1
-      ;;
-  esac
-    shift 1
-done
-
-usage() {
-  echo "Usage: $0 [--server-host <server_host>] [--server-port <server_port>] [--disable_log_file] [--run-bin]"
-}
+echo "Options: $options"
 
 # Function to handle termination
 cleanup() {
   echo "Terminating all start_agent.py processes..."
-  for pid in "${pids[@]}"; do
-    kill $pid
-  done
+  kill $pid
   exit 0
 }
 
 # Trap SIGTERM and SIGINT signals
 trap cleanup SIGTERM SIGINT
 
-# Initialize an empty array to store PIDs
-pids=()
-
-# Set options
-options="--server-host $server_host --server-port $server_port --use-random-port --close-server"
-if [ $disable_log_file = false ]; then
-  # Init log directory
-  log_dir="$(pwd)/logs/$(date +'%Y-%m-%d_%H-%M-%S')_$((RANDOM % 900000 + 100000))"
-  mkdir -p $log_dir
-  options="$options --log-dir $log_dir"
-else
-  options="$options --disable-log-file"
-fi
-
-# Run start.py or start.bin
-if [ $run_bin = true ]; then
-  run_command="./start.bin"
-else
-  # active .venv
-  source $python_env
-  run_command="python3 start.py"
-fi
-
-if [ $separate_rpc_server = true ]; then
-  # Run start_agent.py 11 times and store each PID in the array
-  $run_command $options --goalie &
-  pids+=($!)
-
-  sleep 2
-
-  for i in {2..11}; do
-    $run_command $options --player &
-    pids+=($!)
-  done
+run_command="python3 start.py"
 
-  $run_command $options --coach &
-  pids+=($!)
-else
-  $run_command $options &
-  pids+=($!)
-fi
+$run_command $options &
+pid=$!
 
-# Wait for all background processes to finish
-for pid in "${pids[@]}"; do
-  wait $pid
-done
+# Wait for the process to finish
+wait $pid