use the database for storing / retrieving logs

rootfs/monitor-hub/monitor-hub.py

@@ -14,7 +14,7 @@
 from sys import exit
 import os
 
-from sqlalchemy import create_engine, Column, Integer, String, Text, Date
+from sqlalchemy import create_engine, Column, Integer, String, Text, Date, Float
 from sqlalchemy.sql import text
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.ext.declarative import declarative_base
@@ -60,7 +60,7 @@ class Logs(LogEntries):
     __tablename__ = "logs"
     id = Column(Integer, primary_key=True)
     container = Column(String(255), nullable=False)
-    time = Column(Date, nullable=False)
+    time = Column(Float, nullable=False)
     log = Column(Text, nullable=False)
 
 
@@ -104,13 +104,47 @@ def __hash__(self):
     def __ne__(self, other):
         return not (self == other)
 
+    def clear_logs(self):
+        # clear the logs from the database
+        try:
+            db = db_session()
+            db.query(Logs).filter(Logs.container == self.name).delete()
+            db.commit()
+            db.close()
+        except Exception as e:
+            print(f"Error clearing logs: {e}")
+
     def get_log(self, num_lines=50):
+        results = []
+        # get the logs from the database
+        try:
+            session = db_session()
+            result = (
+                session.query(Logs)
+                .where(Logs.container == self.name)
+                .order_by(Logs.time.asc())
+                .limit(num_lines)
+            )
+
+            # convert the result to a list of dictionaries
+            results = [query_to_dict(x) for x in result]
+
+            for result in results:
+                result.pop("id", None)
+        except Exception as e:
+            print(f"Error getting logs: {e}")
+        finally:
+            return results
+
+    def get_log_short(self, num_lines=50):
         # get the logs from the database
-        session = db_session()
-        result = session.query(Logs).order_by(Logs.time.desc()).limit(50)
+        results = self.get_log(num_lines)
+
+        # remove the `container`   key from the dictionary
+        for result in results:
+            result.pop("container", None)
 
-        # convert the result to a list of dictionaries
-        return [query_to_dict(x) for x in result]
+        return results
 
     def get_name(self):
         return f"{self.name}"
@@ -141,7 +175,9 @@ def container_logs(self, container):
                 time_stamp = time_stamp[:22] + time_stamp[28:]
                 # convert the time stamp to a unix timestamp with milliseconds
 
-                time_stamp = datetime.strptime(time_stamp, "%Y-%m-%dT%H:%M:%S.%f%z")
+                time_stamp = datetime.strptime(
+                    time_stamp, "%Y-%m-%dT%H:%M:%S.%f%z"
+                ).timestamp()
 
                 # send the log entry to the database
                 db = db_session()
@@ -154,7 +190,7 @@ def container_logs(self, container):
                         "new_log",
                         {
                             "name": container.name,
-                            "time": time_stamp.timestamp(),
+                            "time": time_stamp,
                             "log": entry,
                         },
                         namespace="/main",
@@ -173,6 +209,7 @@ def container_logs(self, container):
                 {"name": container.name},
                 namespace="/main",
             )
+
         containers.remove(container.name)
         # remove container from the list of container classes
         container_classes.remove(self)
@@ -237,19 +274,32 @@ def main_connect():
     output = {}
 
     for container in container_classes:
-        print(f"{container.get_log()}")
-        # output[container.name] = {
-        #     "logs": container.get_log(),
-        #     "status": "running",
-        #     "name": container.name,
-        # }
+        output[container.name] = {
+            "logs": container.get_log_short(),
+            "status": "running",
+            "name": container.name,
+        }
 
     socketio.emit("connect_data", output, namespace="/main", to=requester)
 
 
 # main function
 if __name__ == "__main__":
     signal(SIGINT, exit_handler)
+
+    print("Clearing database")
+    try:
+        # make sure database is empty
+        db = db_session()
+        db.query(Logs).delete()
+        db.commit()
+        db.close()
+    except Exception as e:
+        print(f"Error clearing database: {e}")
+        # exit
+        exit(1)
+    print("Database cleared")
+
     # create a thread to check for new containers
     check_thread = Thread(target=pause_and_check)
     check_thread.start()

src/index.ts

@@ -15,21 +15,35 @@ let active_container: string | null = null;
 let should_scroll: boolean = true;
 
 $((): void => {
+  // There may be an issue where the auto scroll in to view is happening when a new log entry comes in.
+  // This may prevent the auto-scrolling from working. Will need more testing.
+  $("#container-logs").on("scroll", function (_) {
+    // Ensure we have a valid scroll_logs element
+    const scroll_logs: JQuery<HTMLElement> = $("#container-logs");
+    const scrollTop: number | undefined = scroll_logs.scrollTop();
+    const scrollHeight: number | undefined = scroll_logs[0].scrollHeight;
+
+    if (!scroll_logs || scrollTop === undefined || scrollHeight === undefined)
+      return;
+
+    // if the user is scrolling unless the user is at the bottom
+    if (scrollHeight + scrollHeight < scroll_logs[0].scrollHeight) {
+      should_scroll = false;
+    } else {
+      should_scroll = true;
+    }
+  });
+
   //connect to the socket server.
   const socket: Socket = io(`${document.location.origin}/main`, {
     path: "/socket.io",
   });
 
-  socket.on("connect", () => {
-    console.log("connected");
-  });
+  socket.on("connect", () => {});
 
-  socket.on("disconnect", () => {
-    console.error("disconnected");
-  });
+  socket.on("disconnect", () => {});
 
   socket.on("container_start", (data: ContainerStart) => {
-    console.log("New Container: ", data.name);
     // add the container to containers
     containers[data.name] = {
       name: data.name,
@@ -40,15 +54,13 @@ $((): void => {
   });
 
   socket.on("connect_data", (data: OnConnectContainersAndLogs) => {
-    console.log("Connect Data: ", data);
     containers = data;
     generate_li_list();
     active_container = get_first_container_sorted();
     show_logs(active_container);
   });
 
   socket.on("container_exit", (data: ContainerExit) => {
-    console.log("Container Exit: ", data);
     // remove the container from containers
     delete containers[data.name];
 
@@ -59,32 +71,10 @@ $((): void => {
     }
   });
 
-  // There may be an issue where the auto scroll in to view is happening when a new log entry comes in.
-  // This may prevent the auto-scrolling from working. Will need more testing.
-  $("#container-logs").on("scroll", function (_) {
-    // Ensure we have a valid scroll_logs element
-    const scroll_logs: JQuery<HTMLElement> = $("#container-logs");
-    const scrollTop: number | undefined = scroll_logs.scrollTop();
-    const scrollHeight: number | undefined = scroll_logs[0].scrollHeight;
-
-    if (!scroll_logs || !scrollTop || !scrollHeight) {
-      console.error("Cannot scroll right now");
-      return;
-    }
-    // if the user is scrolling unless the user is at the bottom
-    if (scrollHeight + scrollHeight < scroll_logs[0].scrollHeight) {
-      should_scroll = false;
-    } else {
-      should_scroll = true;
-    }
-  });
-
   socket.on("new_log", (data: NewLog) => {
     // add the log message to the container
-    if (!containers[data.name]) {
-      console.error("Container not found: ", data.name);
-      return;
-    }
+    if (!containers[data.name]) return;
+
     containers[data.name].logs.push(data);
 
     while (containers[data.name].logs.length > 100) {
@@ -113,8 +103,6 @@ $((): void => {
       }
     }
   });
-
-  console.log("loaded");
 });
 
 function show_logs(name: string) {
@@ -147,7 +135,9 @@ window.show_logs = function (name: string) {
 
 function generate_log_element(log: ShortLogs) {
   // BE CAREFUL HERE. IF YOU CHANGE THE P TAG TO A DIFFERENT TAG, YOU MUST CHANGE THE REMOVE LOGS CODE IN THE NEW_LOG EVENT
-  return `<p>${log.time} | ${stripAnsi(log.log)}</p>`;
+  return `<p>${new Date(log.time * 1000).toLocaleString()} | ${stripAnsi(
+    log.log
+  )}</p>`;
 }
 
 function generate_li_element(name: string) {