MANUAL file expiry deletion script

.github/workflows/unit_tests.yml

@@ -0,0 +1,29 @@
+name: File Auto Expiry Unit Tests
+on: [pull_request]
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Test with pytest
+        run: |
+          pip install pytest pytest-cov
+          pytest source/tests/test_utils.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
+
+

.gitignore

@@ -0,0 +1,177 @@
+# Created by https://www.toptal.com/developers/gitignore/api/python
+# Edit at https://www.toptal.com/developers/gitignore?templates=python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+source/__pycache__
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+# End of https://www.toptal.com/developers/gitignore/api/python

README.md

@@ -1 +1,14 @@
-# infra_file_auto_expiry
+# infra_file_auto_expiry
+
+Relating to issue: https://github.com/WATonomous/infra-config/issues/1143
+
+This project is meant to help automatically expire and delete files. It's currently at the stage of gathering all necessary information about file deletion easier. In the future, it is required to add a notification system for users whose files are to be deleted, and an actual deletion system. 
+
+Currently it moves through every single top level folder in a directory, and checks whether it is expired or not. This means that every single file in that directory tree must be expired. As it does this, it gathers all the users who created files in that directory, and the days since the most RECENT atime, ctime, and mtime of ANY file in that directory. It only collects these for folders which have been confirmed to be expired.
+
+To collect the expiry information of all top level directories in a given path:
+sudo $(which python3)  /path_to_directory/infra_file_auto_expiry/infra_file_auto_expiry/source/main.py collect-file-info path_to_check_expiry_of
+
+This will return a jsonl file. You can then use this in the following command to tabulate all expired paths that are associated with a particular user. 
+
+sudo $(which python3)  /path_to_directory/infra_file_auto_expiry/infra_file_auto_expiry/source/main.py collect-creator-info path_to_jsonl_file

requirements.txt

@@ -0,0 +1 @@
+typer>=0.12.3

source/data/expiry_constants.py

@@ -0,0 +1,18 @@
+# Ignore the following directories because they contain
+# large number of generated files. Files in ignored
+# directories are assumed to have not been accessed recently.
+DIRECTORIES_TO_IGNORE = {
+    "ros/humble",
+
+    "lib/python3.10",
+    "lib/python3.9",
+    "lib/python3.8",
+    "lib/python3.7",
+
+    "lib64/python3.10",
+    "lib64/python3.9",
+    "lib64/python3.8",
+    "lib64/python3.7",
+}
+
+SECS_PER_DAY = 86400

source/data/tuples.py

@@ -0,0 +1,4 @@
+from collections import namedtuple
+
+expiry_tuple = namedtuple("file_tuple", "is_expired, creators, atime, ctime, mtime")
+creator_tuple = namedtuple("creator_tuple", "username, uid, gid")

source/main.py

@@ -0,0 +1,34 @@
+from utils.interface import *
+from data.expiry_constants import SECS_PER_DAY
+import time
+import typer
+app = typer.Typer()
+
+@app.command()
+def collect_file_info(path: str, save_file: str = "", days_for_expiry: int = 10):
+    """
+    Collects information about the top level paths within a given folder path
+    And dumps it into a json file, specified by the save_file flag
+    """
+    scrape_time = time.time()
+    seconds_for_expiry = int(days_for_expiry) * SECS_PER_DAY
+    expiry_threshold = scrape_time - seconds_for_expiry
+    collect_expired_file_information(folder_path=path, 
+                                     save_file=save_file, 
+                                     scrape_time=scrape_time, 
+                                     expiry_threshold=expiry_threshold)
+
+@app.command()
+def collect_creator_info(file_info: str, save_file: str = ""):
+    """
+    Tabulates the paths that relate to specific users, based on a given jsonl path
+    That jsonl path should be the result of calling the collect_file_info function
+    It then dumps the new information into another json file, specified by the save_file flag
+    """
+    scrape_time = time.time()
+    collect_creator_information(path_info_file=file_info, 
+                                save_file=save_file, 
+                                scrape_time=scrape_time)
+
+if __name__ == "__main__":
+    app()

source/tests/test_utils.py

@@ -0,0 +1,94 @@
+import unittest
+import os
+import sys
+from unittest.mock import MagicMock, patch
+module_path = os.path.dirname(
+    os.path.dirname(os.path.abspath(__file__))
+)
+sys.path.append(module_path)
+
+from utils.interface import *
+from utils.expiry_checks import *
+
+class TestUtils(unittest.TestCase):
+    @patch("pwd.getpwuid")
+    @patch("os.stat")
+    def test_get_file_creator(self, patch_stat, patch_pwd):
+        """
+        Tests retrieving the user name of a file owner
+        """
+        # Successfully retrieves file owner
+        patch_stat.return_value.st_uid=5111
+        patch_stat.return_value.st_gid=1555
+        patch_pwd.return_value.pw_name="tester_account"
+
+        file_creator = get_file_creator("/home/machung/test.txt")
+        self.assertEqual(file_creator[0], "tester_account")
+        self.assertEqual(file_creator[1], 5111)
+        self.assertEqual(file_creator[2], 1555)
+
+    @patch('os.stat')
+    def test_is_expired_filepath(self, patch_stat):
+        """
+        Tests the is_expired_file function
+        """
+        time_for_expiry = 30 # 30 days
+        patch_stat.st_atime = 5 # 5 days
+        patch_stat.st_ctime = 5 # 5 days
+        patch_stat.st_mtime = 5 # 5 days
+        scrape_time = 50  # 50 days
+        expiry_threshold = scrape_time - time_for_expiry
+
+        # Days since last access is 5 < 20
+        # The file should be expired
+        self.assertTrue(is_expired_filepath("test_name.txt", patch_stat, expiry_threshold)[0])
+
+        expiry_threshold = -20  # change to 10 days
+        # Days since last access is 5 > -20
+        # The file should not be expired
+        expiry_test_result =  is_expired_filepath("test_name.txt", patch_stat, expiry_threshold)
+        self.assertFalse(expiry_test_result[0])
+        self.assertTrue(5, expiry_test_result[2])
+        self.assertTrue(5, expiry_test_result[3])
+        self.assertTrue(5, expiry_test_result[4])
+
+    @patch('os.listdir')
+    @patch("os.stat")
+    @patch("utils.expiry_checks.is_expired")
+    def test_is_expired_folder(self, patch_expired, patch_stat, patch_path):
+        """
+        Tests the is_expired_folder function. This should return 
+        True (is_expired) if all subdirectories and files are also expired. 
+
+        The values of atime, ctime, and mtime should be the largest timestamps 
+        seen from the entire folder tree. This indicates the most recent timestamp. 
+        In the test we just simulate those timestamps by using smaller integers. 
+        """
+        mocked_file_expiry_results_1 = MagicMock()
+        mocked_file_expiry_results_2 = MagicMock()
+
+        mocked_file_expiry_results_1.configure_mock(
+            is_expired = True, creators = ("a", 0, 0), atime = 1000, 
+            ctime = 2000, mtime = 10000)
+            # atime, ctime, mtime = 5, 7, and 10 days respectively
+
+        mocked_file_expiry_results_2.configure_mock(
+            is_expired = False, creators = ("b", 1, 1), atime = 2000, 
+            ctime = 6000 , mtime = 5000)
+           # atime, ctime, mtime = 7, 6, and 15 days respectively
+
+        patch_expired.side_effect = [mocked_file_expiry_results_1, 
+                                     mocked_file_expiry_results_2]
+        patch_path.return_value = ["one.txt", "two.txt"]
+
+        # atime, ctime, mtime for the folder itself is 5 days for all
+        patch_stat.st_atime = patch_stat.st_ctime = patch_stat.st_mtime = 3000
+
+        res = is_expired_folder("test_path", patch_stat, 0)
+        self.assertEqual(False, res[0])
+        self.assertEqual(3000 , res[2])
+        self.assertEqual(6000 , res[3])
+        self.assertEqual(10000 , res[4])
+
+if __name__ == '__main__':
+    unittest.main()