User recording

README.md

@@ -14,16 +14,21 @@ The environment will eventually provide 4 modes of operation for players and age
 Minetester
 ==========
 Minetester is the Python package that exposes Minetest environments via the `gym(nasium)` interface.
-After [building the minetest executable](https://github.com/EleutherAI/minetest/blob/develop/build_instructions.txt) you can install it with:
+
+The library provides the minetester environment you can inspect the [test loop](https://github.com/EleutherAI/minetest/blob/develop/minetester/scripts/test_loop.py) to see how to run the environment.
+
+This script can be run with:
 ```
-pip install -e .
+python -m minetester.scripts.test_loop
 ```
-To verify the installation run
+
+The installation also provides a script for recording the game with:
+
 ```
-python -m minetester.scripts.test_loop
+python -m minetester.scripts.record_game --data_dir <recording_directory>
 ```
 
-Quick Build Instructions for Linux
+Quick Build Instructions for Ubuntu 20/22.04 and x86
 ==================================
 
 Run these make commands in order to build and install minetester. 
@@ -52,6 +57,11 @@ make install #install python library into local environment along with nessesary
 make demo #run the demo script
 ```
 
+Note: it is possible to run these instructions with ubuntu 20.04, with the following modifications due to the reliance on [auditwheel](https://github.com/pypa/auditwheel)
+
+ - patchelf must be installed using something other than the package manager. The conda-forge distribution seems to work well.
+ - by default `make minetester` and `make install` assume  `manylinux_2_35_x86_64` will work, this must be downgraded to `manylinux_2_31_x86_64`
+
 Minetest
 ========
 

minetester/data_recorder.py

@@ -0,0 +1,105 @@
+import os
+import time
+import json
+import cv2
+import zmq
+import numpy as np
+from minetester.utils import unpack_pb_obs
+
+class DataRecorder:
+    def __init__(
+        self,
+        data_path: os.PathLike,
+        target_address: str,
+        timeout: int = 1000,
+        max_queue_length: int = 1200,
+        max_attempts: int = 10,
+        debug: bool = False,
+        subsample_rate: float = 0.1,  # time in seconds between recorded frames
+        frame_shape = (512,300)
+    ):
+        self.target_address = target_address
+        self.data_path = data_path
+        self.timeout = timeout
+        self.max_queue_length = max_queue_length
+        self.max_attempts = max_attempts
+        self.debug = debug
+        self.subsample_rate = subsample_rate  # time between frames to record
+        self.frame_shape = frame_shape
+        self._recording = False
+
+        # Setup ZMQ
+        self.context = zmq.Context()
+        self.socket = self.context.socket(zmq.SUB)
+        self.socket.RCVTIMEO = self.timeout
+        self.socket.connect(f"tcp://{self.target_address}")
+
+        # Subscribe to all topics
+        self.socket.setsockopt(zmq.SUBSCRIBE, b"")
+
+        # Set maximum message queue length (high water mark)
+        self.socket.setsockopt(zmq.RCVHWM, self.max_queue_length)
+
+        # Set timeout in milliseconds
+        self.socket.setsockopt(zmq.RCVTIMEO, 1000)
+
+        # Create data directory if it doesn't exist
+        os.makedirs(self.data_path, exist_ok=True)
+
+        # Prepare paths for video and action log
+        self.video_path = os.path.join(self.data_path, "video.avi")
+        self.action_log_path = os.path.join(self.data_path, "actions.jsonl")
+
+        # Initialize video writer
+        self.fourcc = cv2.VideoWriter_fourcc(*'XVID')
+        self.video_writer = cv2.VideoWriter(self.video_path, self.fourcc, 1/self.subsample_rate, frame_shape)
+
+    def start(self):
+        with open(self.action_log_path, "w") as action_log:
+            self._recording = True
+            num_attempts = 0
+            next_frame_time = time.time()  # time for the next frame to record
+
+            while self._recording:
+                try:
+                    # Receive data
+                    raw_data = self.socket.recv()
+                    num_attempts = 0
+
+                    # Check if it's time to record a frame
+                    if time.time() >= next_frame_time:
+                        obs, rew, terminal, info, action = unpack_pb_obs(raw_data)
+
+                        if self.debug:
+                            print(obs, type(obs))
+                            action_str = ""
+                            for key in action.keys():
+                                if key != "MOUSE" and action[key]:
+                                    action_str += key + ", "
+                            print(f"action={action_str}, rew={rew}, T?={terminal}")
+
+                        # Write frame to video
+                        resized_frame = cv2.resize(np.array(obs), self.frame_shape)
+                        self.video_writer.write(cv2.cvtColor(resized_frame, cv2.COLOR_RGB2BGR))
+
+                        # Write action to action log (one line per frame)
+                        action_log.write(json.dumps(action) + "\n")
+
+                        # Update time for the next frame
+                        next_frame_time += self.subsample_rate
+
+                except zmq.ZMQError as err:
+                    if err.errno == zmq.EAGAIN:
+                        print(f"Reception attempts: {num_attempts}")
+                        if num_attempts >= self.max_attempts:
+                            print("Session finished.")
+                            self._recording = False
+                        num_attempts += 1
+                    else:
+                        print(f"ZMQError: {err}")
+                        self._recording = False
+
+    def stop(self):
+        self._recording = False
+        self.video_writer.release()  # release video writer
+

minetester/scripts/record_game.py

@@ -0,0 +1,48 @@
+import subprocess
+import sys
+import argparse
+from minetester.data_recorder import DataRecorder
+import multiprocessing
+
+def run_data_recorder(debug, data_dir):
+    # Start recorder
+    address = "localhost:5555"
+    num_attempts = 10
+    recorder = DataRecorder(data_dir, address, max_attempts=num_attempts, debug=debug)
+    recorder.start()  # warning: file quickly grows very large
+
+def run_minetest():
+    command = [
+        "bin/minetest",
+        "--name", "MinetestAgent",
+        "--password", "password",
+        #"--address", "0.0.0.0",
+        #"--port", "30000",
+        "--go",
+        "--client-address", "tcp://*:5555",
+        "--record",
+        "--noresizing",
+        "--cursor-image", "cursors/mouse_cursor_white_16x16.png",
+        "--config", "scripts/minetest.conf",
+    ]
+
+    try:
+        subprocess.run(command, check=True)
+    except subprocess.CalledProcessError as e:
+        print(f"Command '{' '.join(command)}' failed with return code {e.returncode}", file=sys.stderr)
+        sys.exit(1)
+
+def main():
+    parser = argparse.ArgumentParser(description='Launch data recorder and minetest.')
+    parser.add_argument('--debug', action='store_true', help='If set, data is not written')
+    parser.add_argument('--data_dir', type=str, default="user_recording", help='Directory for data recording')
+    args = parser.parse_args()
+
+    process = multiprocessing.Process(target=run_minetest)
+
+    process.start()
+    run_data_recorder(args.debug, args.data_dir)
+    process.join()
+
+if __name__ == "__main__":
+    main()

setup.py

@@ -8,6 +8,7 @@
     author_email='',
     packages=find_packages(),
     install_requires=[
+        'opencv-python',
         'gymnasium',
         'numpy',
         'matplotlib',