Filesystem / metadata changes to better support metadata cache

Expand support for the metadata cache introduced in
1350ab0 by having file providers for
individual files -- previously it was only used for scandir() results.

rime/filesystem/android.py

@@ -8,25 +8,24 @@
 
 import fs.osfs
 
-from .base import DeviceFilesystem, DirEntry
+from .base import DeviceFilesystem
 from .devicesettings import DeviceSettings
 from .fslibfilesystem import FSLibFilesystem
-from .metadata import MetadataDb
 
 
 class AndroidDeviceFilesystem(DeviceFilesystem):
     def __init__(self, id_: str, root: str, metadata_db_path: str):
         self.id_ = id_
-        self._fs = fs.osfs.OSFS(root)
         self._settings = DeviceSettings(root)
-        self._fsaccess = FSLibFilesystem(self._fs, metadata_db_path)
+        self._fsaccess = FSLibFilesystem(fs.osfs.OSFS(root), metadata_db_path)
 
     @classmethod
     def is_device_filesystem(cls, path):
         return os.path.exists(os.path.join(path, 'data', 'data', 'android'))
 
     @classmethod
-    def create(cls, id_: str, root: str, metadata_db_path: str, template: Optional[DeviceFilesystem] = None) -> DeviceFilesystem:
+    def create(cls, id_: str, root: str, metadata_db_path: str, template: Optional[DeviceFilesystem] = None)\
+            -> DeviceFilesystem:
         if os.path.exists(root):
             raise FileExistsError(root)
 
@@ -37,6 +36,10 @@ def create(cls, id_: str, root: str, metadata_db_path: str, template: Optional[D
         obj._settings.set_subset_fs(True)
         return obj
 
+    @property
+    def metadata(self):
+        return self._fsaccess.metadata
+
     def dirname(self, pathname):
         return self._fsaccess.dirname(pathname)
 
@@ -49,20 +52,17 @@ def stat(self, pathname):
     def is_subset_filesystem(self) -> bool:
         return self._settings.is_subset_fs()
 
-    def path_to_direntry(self, path) -> DirEntry:
-        return self._fsaccess.path_to_direntry(path)
-
     def scandir(self, path):
         return self._fsaccess.scandir(path)
 
     def exists(self, path):
-        return self._fs.exists(path)
+        return self._fsaccess.exists(path)
 
     def getsize(self, path):
-        return self._fs.getsize(path)
+        return self._fsaccess.getsize(path)
 
     def open(self, path):
-        return self._fs.open(path, 'rb')
+        return self._fsaccess.open(path)
 
     def create_file(self, path):
         return self._fsaccess.create_file(path)
@@ -79,6 +79,9 @@ def lock(self, locked: bool):
     def is_locked(self) -> bool:
         return self._settings.is_locked()
 
+    def get_dir_entry(self, path):
+        return self._fsaccess.get_dir_entry(path)
+
 
 class AndroidZippedDeviceFilesystem(DeviceFilesystem):
     """
@@ -128,15 +131,13 @@ def is_device_filesystem(cls, path):
             return zipfile.Path(zp, os.path.join(main_dir.name, 'data', 'data', 'android/')).exists()
 
     @classmethod
-    def create(cls, id_: str, root: str, metadata_db_path: str, template: Optional['DeviceFilesystem'] = None) -> 'DeviceFilesystem':
+    def create(cls, id_: str, root: str, metadata_db_path: str, template: Optional['DeviceFilesystem'] = None)\
+            -> 'DeviceFilesystem':
         return AndroidDeviceFilesystem.create(id_, root, metadata_db_path)
 
     def is_subset_filesystem(self) -> bool:
         return self._settings.is_subset_fs()
 
-    def path_to_direntry(self, path, name=None) -> DirEntry:
-        return self._fsaccess.path_to_direntry(path, name)
-
     def scandir(self, path):
         return self._fsaccess.scandir(path)
 
@@ -172,3 +173,6 @@ def basename(self, pathname):
 
     def stat(self, pathname):
         return self._fsaccess.stat(pathname)
+
+    def get_dir_entry(self, path):
+        return self._fsaccess.get_dir_entry(path)

rime/filesystem/base.py

@@ -4,7 +4,6 @@
 
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-import os
 from typing import Optional, Any, TYPE_CHECKING
 
 from .direntry import DirEntry
@@ -39,7 +38,8 @@ def is_device_filesystem(cls, path) -> bool:
 
     @classmethod
     @abstractmethod
-    def create(cls, id_: str, root: str, metadata_db_path: str, template: Optional['DeviceFilesystem'] = None) -> 'DeviceFilesystem':
+    def create(cls, id_: str, root: str, metadata_db_path: str, template: Optional['DeviceFilesystem'] = None)\
+            -> 'DeviceFilesystem':
         """
         Create a new filesystem of this type at 'path'.
         """
@@ -66,6 +66,13 @@ def scandir(self, path) -> list[DirEntry]:
         """
         return []
 
+    @abstractmethod
+    def get_dir_entry(self, path) -> DirEntry:
+        """
+        Returns a DirEntry (a stat-like object with MIME type) using the metadata database.
+        """
+        raise NotImplementedError()
+
     @abstractmethod
     def exists(self, path) -> bool:
         """
@@ -141,10 +148,6 @@ def basename(self, pathname):
     def stat(self, pathname):
         raise NotImplementedError(pathname)
 
-    @abstractmethod
-    def path_to_direntry(self, path) -> DirEntry:
-        raise NotImplementedError()
-
     def is_encrypted(self) -> bool:
         return False
 

rime/filesystem/direntry.py

@@ -18,6 +18,7 @@
 # Just kidding, chosen by reference to https://github.com/h2non/filetype.py
 FILE_HEADER_GUESS_LENGTH = 261
 
+
 @dataclass(eq=True, frozen=True)
 class DirEntry:
     """
@@ -56,5 +57,5 @@ def from_path(cls, fs, path):
                     mime_type = filetype.mime
         else:
             mime_type = MIME_TYPE_CANNOT_DETERMINE
-        
+
         return cls(path, stat_val, mime_type)

rime/filesystem/fslibfilesystem.py

@@ -1,6 +1,5 @@
 import os
 
-from .base import DirEntry
 from .ensuredir import ensuredir
 from . import metadata
 from ..sql import sqlite3_connect_filename as sqlite3_connect_with_regex_support
@@ -12,7 +11,7 @@
 class FSLibFilesystem:
     def __init__(self, _fs, metadata_db_path):
         self._fs = _fs
-        self._metadata = metadata.MetadataDb(metadata_db_path)
+        self.metadata = metadata.MetadataDb(metadata_db_path)
 
     def dirname(self, pathname):
         if '/' not in pathname:
@@ -26,20 +25,22 @@ def basename(self, pathname):
 
         return pathname[pathname.rindex('/') + 1:]
 
+    def exists(self, path):
+        return self._fs.exists(path)
+
+    def getsize(self, path):
+        return self._fs.getsize(path)
+
     def open(self, path):
         return self._fs.open(path, 'rb')
 
     def stat(self, pathname):
         return os.stat(self._fs.getsyspath(pathname))
 
-    def path_to_direntry(self, path, name=None) -> DirEntry:
-        return DirEntry.from_path(self, path)
-
     def scandir(self, path):
-        result = []
         pathnames = [os.path.join(path, name) for name in self._fs.listdir(path)]
 
-        return metadata.get_dir_entries_and_update_db(self, self._metadata, pathnames)
+        return metadata.get_dir_entries_and_update_db(self, self.metadata, pathnames)
 
     def create_file(self, path):
         ensuredir(self._fs.getsyspath(path))
@@ -55,3 +56,6 @@ def sqlite3_create(self, path):
         ensuredir(syspath)
 
         return sqlite3_connect_with_regex_support(syspath, read_only=False)
+
+    def get_dir_entry(self, path):
+        return metadata.get_dir_entry_and_update_db(self, self.metadata, path)

rime/filesystem/ios.py

@@ -195,7 +195,8 @@ def is_device_filesystem(cls, path):
         )
 
     @classmethod
-    def create(cls, id_: str, root: str, metadata_db_path: str, template: Optional['DeviceFilesystem'] = None) -> 'DeviceFilesystem':
+    def create(cls, id_: str, root: str, metadata_db_path: str, template: Optional['DeviceFilesystem'] = None)\
+            -> 'DeviceFilesystem':
         if os.path.exists(root):
             raise FileExistsError(root)
 
@@ -243,6 +244,9 @@ def sqlite3_connect(self, path, read_only=True):
     def scandir(self, path):
         return self._converter.scandir(path)
 
+    def get_dir_entry(self, path):
+        raise NotImplementedError()
+
     def exists(self, path):
         real_path = self._converter.get_hashed_pathname(path)
         return os.path.exists(os.path.join(self.root, real_path))
@@ -291,9 +295,6 @@ def basename(self, path):
     def stat(self, pathname):
         raise NotImplementedError(pathname)
 
-    def path_to_direntry(self, path):
-        raise NotImplementedError
-
 
 class IosZippedDeviceFilesystem(DeviceFilesystem, IosDeviceFilesystemBase):
     """
@@ -366,7 +367,8 @@ def is_device_filesystem(cls, path) -> bool:
             )
 
     @classmethod
-    def create(cls, id_: str, root: str, metadata_db_path, template: Optional['DeviceFilesystem'] = None) -> 'DeviceFilesystem':
+    def create(cls, id_: str, root: str, metadata_db_path, template: Optional['DeviceFilesystem'] = None)\
+            -> 'DeviceFilesystem':
         return IosDeviceFilesystem.create(id_, root, metadata_db_path, template=template)
 
     def is_subset_filesystem(self) -> bool:
@@ -375,6 +377,9 @@ def is_subset_filesystem(self) -> bool:
     def scandir(self, path) -> list[DirEntry]:
         return self._converter.scandir(path)
 
+    def get_dir_entry(self, path):
+        raise NotImplementedError()
+
     def exists(self, path) -> bool:
 
         real_path = self._converter.get_hashed_pathname(path)
@@ -446,9 +451,6 @@ def basename(self, path):
     def stat(self, pathname):
         raise NotImplementedError(pathname)
 
-    def path_to_direntry(self, path):
-        raise NotImplementedError
-
 
 class IosEncryptedDeviceFilesystem(DeviceFilesystem):
 
@@ -497,6 +499,9 @@ def is_subset_filesystem(self) -> bool:
     def scandir(self, path) -> list[str]:
         return []
 
+    def get_dir_entry(self, path):
+        raise NotImplementedError()
+
     def listdir(self, path) -> list[str]:
         if self.manifest is None:
             raise NotDecryptedError()
@@ -656,6 +661,3 @@ def basename(self, path):
 
     def stat(self, pathname):
         raise NotImplementedError(pathname)
-
-    def path_to_direntry(self, path):
-        raise NotImplementedError

rime/filesystem/metadata.py

@@ -6,11 +6,7 @@
 import os
 import pickle
 
-from dataclasses import dataclass
-
-from .direntry import DirEntry
-
-from ..sql import sqlite3_connect_filename, Table, Query, Column, Parameter
+from ..sql import sqlite3_connect_filename, Table, Query, Parameter, Column
 from .direntry import DirEntry
 
 
@@ -28,15 +24,25 @@ def _init_db(self, db_pathname):
 
         conn = sqlite3_connect_filename(db_pathname, read_only=False)
 
-        query = "CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value TEXT)"
-        conn.execute(query)
-
-        query = "CREATE TABLE IF NOT EXISTS mime_types (id INTEGER PRIMARY KEY AUTOINCREMENT, mime_type TEXT)"
-        conn.execute(query)
-
-        query = "CREATE TABLE IF NOT EXISTS dir_entries (" +\
-            "id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT, mime_type_id INTEGER, stat_val TEXT)"
-        conn.execute(query)
+        query = Query.create_table(Table('settings')).if_not_exists().columns(
+            Column('key', 'TEXT'),
+            Column('value', 'TEXT'))\
+            .primary_key('key')
+        conn.execute(query.get_sql())
+
+        query = Query.create_table(Table('mime_types')).if_not_exists().columns(
+            Column('id', 'INT'),
+            Column('mime_type', 'TEXT'))\
+            .primary_key('id')
+        conn.execute(query.get_sql())
+
+        query = Query.create_table(Table('dir_entries')).if_not_exists().columns(
+            Column('id', 'INT'),
+            Column('path', 'TEXT'),
+            Column('mime_type_id', 'INT'),
+            Column('stat_val', 'TEXT'))\
+            .primary_key('id')
+        conn.execute(query.get_sql())
 
         return conn
 
@@ -111,7 +117,10 @@ def add_dir_entries_for_pathnames(self, dir_entries: list[DirEntry]):
         results = self.db.execute(str(query)).fetchall()
 
         mime_type_ids = {result[1]: result[0] for result in results}  # map mime type to ID
-        mime_types_to_add = [dir_entry.mime_type for dir_entry in dir_entries if dir_entry.mime_type not in mime_type_ids]
+        mime_types_to_add = [
+            dir_entry.mime_type
+            for dir_entry in dir_entries
+            if dir_entry.mime_type not in mime_type_ids]
 
         if mime_types_to_add:
             query = Query\
@@ -155,8 +164,19 @@ def get_dir_entries_and_update_db(fs, metadata_db, pathnames: list[str]) -> list
     pathnames_to_add = [pathname for pathname in pathnames if pathname not in found_pathnames]
 
     if pathnames_to_add:
-        dir_entries_to_add = [fs.path_to_direntry(pathname) for pathname in pathnames_to_add]
+        dir_entries_to_add = [DirEntry.from_path(fs, pathname) for pathname in pathnames_to_add]
         metadata_db.add_dir_entries_for_pathnames(dir_entries_to_add)
         dir_entries.extend(dir_entries_to_add)
 
     return dir_entries
+
+
+def get_dir_entry_and_update_db(s, metadata_db, pathname: str) -> DirEntry:
+    """
+    Convenient wrapper around get_dir_entries_and_update_db() for a single pathname.
+    """
+    dir_entries = get_dir_entries_and_update_db(s, metadata_db, [pathname])
+    if len(dir_entries) != 1:
+        raise FileNotFoundError(pathname)
+
+    return dir_entries[0]

rime/providers/androidgenericmedia.py

@@ -108,7 +108,7 @@ def get_media(self, local_id) -> MediaData:
         """
         return a MediaData object supplying the picture, video, sound, etc identified by 'local_id'.
         """
-        direntry = self.fs.path_to_direntry(local_id)   # TODO use DB
+        direntry = self.fs.get_dir_entry(local_id)
 
         return MediaData(
             mime_type=direntry.mime_type,