fix: prevent downloading resources more than once (#160)

* umu_runtime: create a lock when downloading runtime * umu_proton: create a lock when downloading proton * requirements: add filelock * workflows: install filelock * Makefile: ensure pip3 when installing dependencies * Makefile: install filelock * packaging: add filelock dependency * deb: fix filelock dependency * umu: update filelock debug statements * pyproject: add filelock dependency * pyproject: ignore build directories * flatpak: build filelock dependencies * flatpak: fix build * flatpak: fix filelock build * flatpak: remove hatchling build dependencies - Some of them will already be built once filelock is built * flatpak: update manifest
Open-Wine-Components · Jul 28, 2024 · a505a90 · a505a90
1 parent db063a0
commit a505a90
Show file tree

Hide file tree

Showing 12 changed files with 141 additions and 64 deletions.
diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml
@@ -17,7 +17,7 @@ jobs:
     - uses: actions/checkout@v4
     - name: Install dependencies
       run: |
-        sudo apt-get install meson shellcheck scdoc python3-hatchling python3-build python3-installer
+        sudo apt-get install meson shellcheck scdoc python3-hatchling python3-build python3-installer python3-filelock
     - name: Initialize submodules
       run: |
         git submodule update --init --recursive

diff --git a/.github/workflows/umu-python.yml b/.github/workflows/umu-python.yml
@@ -30,6 +30,7 @@ jobs:
         python3 -m pip install --upgrade pip
         pip install ruff
         pip install python-xlib
+        pip install filelock
     - name: Lint umu_*.py files with Ruff
       run: |
         pip install ruff

diff --git a/Makefile.in b/Makefile.in
@@ -127,17 +127,18 @@ umu-launcher-install: umu-launcher-dist-install umu-launcher-bin-install
 
 $(OBJDIR)/.build-umu-subprojects: | $(OBJDIR)
 	$(info :: Building subprojects )
-	pip install -r requirements.in -t $(OBJDIR)
+	pip3 install -r requirements.in -t $(OBJDIR)
 
 .PHONY: umu-subprojects
 umu-subprojects: $(OBJDIR)/.build-umu-subprojects
 
 umu-subprojects-install:
 	$(info :: Installing subprojects )
 	install -d $(DESTDIR)$(PYTHONDIR)
-	cp      -r $(OBJDIR)/*-info $(DESTDIR)$(PYTHONDIR)
-	cp      -r $(OBJDIR)/Xlib   $(DESTDIR)$(PYTHONDIR)
-	cp         $(OBJDIR)/six.py $(DESTDIR)$(PYTHONDIR)
+	cp      -r $(OBJDIR)/*-info     $(DESTDIR)$(PYTHONDIR)
+	cp      -r $(OBJDIR)/Xlib       $(DESTDIR)$(PYTHONDIR)
+	cp      -r $(OBJDIR)/filelock   $(DESTDIR)$(PYTHONDIR)
+	cp         $(OBJDIR)/six.py     $(DESTDIR)$(PYTHONDIR)
 
 $(OBJDIR):
 	@mkdir -p $(@)

diff --git a/packaging/deb/debian/control b/packaging/deb/debian/control
@@ -34,5 +34,6 @@ Depends:
  ${shlibs:Depends},
  ${misc:Depends},
  python3,
- python3-xlib (>= 0.33)
+ python3-xlib (>= 0.33),
+ python3-filelock (>= 3.13.1)
 Description: A tool for launching non-steam games with proton.
diff --git a/packaging/flatpak/org.openwinecomponents.umu.umu-launcher.yml b/packaging/flatpak/org.openwinecomponents.umu.umu-launcher.yml
@@ -266,6 +266,7 @@ modules:
                   - type: archive
                     url: https://files.pythonhosted.org/packages/c0/3f/d7af728f075fb08564c5949a9c95e44352e23dee646869fa104a3b2060a3/tomli-2.0.1.tar.gz
                     sha256: de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
+
   # --- python-xlib ---
   - name: python-xlib
     buildsystem: simple
@@ -275,6 +276,72 @@ modules:
       - type: archive
         url: https://files.pythonhosted.org/packages/source/p/python-xlib/python-xlib-0.33.tar.gz
         sha256: 55af7906a2c75ce6cb280a584776080602444f75815a7aff4d287bb2d7018b32
+
+  # --- filelock ---
+  - name: filelock
+    buildsystem: simple
+    build-commands:
+      - pip3 install --no-index --no-build-isolation --prefix="${FLATPAK_DEST}" .
+    sources:
+      - type: archive
+        url: https://files.pythonhosted.org/packages/08/dd/49e06f09b6645156550fb9aee9cc1e59aba7efbc972d665a1bd6ae0435d4/filelock-3.15.4.tar.gz
+        sha256: 2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb
+    modules:
+      # --- python-hatchling ---
+      - name: python-hatchling
+        buildsystem: simple
+        build-commands:
+          - pip3 install --no-index --no-build-isolation --prefix="${FLATPAK_DEST}" .
+        cleanup: ['*']
+        sources:
+          - type: archive
+            url: https://files.pythonhosted.org/packages/a3/51/8a4a67a8174ce59cf49e816e38e9502900aea9b4af672d0127df8e10d3b0/hatchling-1.25.0.tar.gz
+            sha256: 7064631a512610b52250a4d3ff1bd81551d6d1431c4eb7b72e734df6c74f4262
+        modules:
+          # --- pathspec ---
+          - name: pathspec
+            buildsystem: simple
+            build-commands:
+              - pip3 install --no-index --no-build-isolation --prefix="${FLATPAK_DEST}" .
+            cleanup: ['*']
+            sources:
+              - type: archive
+                url: https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz
+                sha256: a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712
+
+          # --- pluggy ---
+          - name: pluggy
+            buildsystem: simple
+            build-commands:
+              - pip3 install --no-index --no-build-isolation --prefix="${FLATPAK_DEST}" .
+            cleanup: ['*']
+            sources:
+              - type: archive
+                url: https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz
+                sha256: 2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1
+
+          # --- trove_classifiers ---
+          - name: trove_classifiers
+            buildsystem: simple
+            build-commands:
+              - pip3 install --no-index --no-build-isolation --prefix="${FLATPAK_DEST}" .
+            cleanup: ['*']
+            sources:
+              - type: archive
+                url: https://files.pythonhosted.org/packages/78/c9/83f915c3f6f94f4c862c7470284fd714f312cce8e3cf98361312bc02493d/trove_classifiers-2024.7.2.tar.gz
+                sha256: 8328f2ac2ce3fd773cbb37c765a0ed7a83f89dc564c7d452f039b69249d0ac35
+
+      # --- hatch-vcs ---
+      - name: hatch-vcs
+        buildsystem: simple
+        build-commands:
+          - pip3 install --no-index --no-build-isolation --prefix="${FLATPAK_DEST}" .
+        cleanup: ['*']
+        sources:
+          - type: archive
+            url: https://files.pythonhosted.org/packages/f5/c9/54bb4fa27b4e4a014ef3bb17710cdf692b3aa2cbc7953da885f1bf7e06ea/hatch_vcs-0.4.0.tar.gz
+            sha256: 093810748fe01db0d451fabcf2c1ac2688caefd232d4ede967090b1c1b07d9f7
+
   # --- umu ---
   # We use make first to install things like version and /usr/share/ files
   # We then use pip to overwrite the python-specific installed files to verify
@@ -292,8 +359,8 @@ modules:
         pip3 install --no-index --no-build-isolation --prefix="${FLATPAK_DEST}" .
     sources:
       - type: git
-        url: https://github.com/Open-Wine-Components/umu-launcher.git
-        branch: main
+        url: https://github.com/R1kaB3rN/ULWGL-launcher.git
+        branch: filelock
     modules:
       # --- packaging ---
       - name: packaging
@@ -306,39 +373,6 @@ modules:
             url: https://files.pythonhosted.org/packages/51/65/50db4dda066951078f0a96cf12f4b9ada6e4b811516bf0262c0f4f7064d4/packaging-24.1.tar.gz
             sha256: 026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002
 
-      # --- trove_classifiers ---
-      - name: trove_classifiers
-        buildsystem: simple
-        build-commands:
-          - pip3 install --no-index --no-build-isolation --prefix="${FLATPAK_DEST}" .
-        cleanup: ['*']
-        sources:
-          - type: archive
-            url: https://files.pythonhosted.org/packages/78/c9/83f915c3f6f94f4c862c7470284fd714f312cce8e3cf98361312bc02493d/trove_classifiers-2024.7.2.tar.gz
-            sha256: 8328f2ac2ce3fd773cbb37c765a0ed7a83f89dc564c7d452f039b69249d0ac35
-
-      # --- pathspec ---
-      - name: pathspec
-        buildsystem: simple
-        build-commands:
-          - pip3 install --no-index --no-build-isolation --prefix="${FLATPAK_DEST}" .
-        cleanup: ['*']
-        sources:
-          - type: archive
-            url: https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz
-            sha256: a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712
-
-      # --- pluggy ---
-      - name: pluggy
-        buildsystem: simple
-        build-commands:
-          - pip3 install --no-index --no-build-isolation --prefix="${FLATPAK_DEST}" .
-        cleanup: ['*']
-        sources:
-          - type: archive
-            url: https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz
-            sha256: 2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1
-
       # --- pyproject_hooks ---
       - name: pyproject_hooks
         buildsystem: simple
@@ -371,14 +405,3 @@ modules:
           - type: archive
             url: https://files.pythonhosted.org/packages/05/18/ceeb4e3ab3aa54495775775b38ae42b10a92f42ce42dfa44da684289b8c8/installer-0.7.0.tar.gz
             sha256: a26d3e3116289bb08216e0d0f7d925fcef0b0194eedfa0c944bcaaa106c4b631
-
-      # --- python-hatchling ---
-      - name: python-hatchling
-        buildsystem: simple
-        build-commands:
-          - pip3 install --no-index --no-build-isolation --prefix="${FLATPAK_DEST}" .
-        cleanup: ['*']
-        sources:
-          - type: archive
-            url: https://files.pythonhosted.org/packages/a3/51/8a4a67a8174ce59cf49e816e38e9502900aea9b4af672d0127df8e10d3b0/hatchling-1.25.0.tar.gz
-            sha256: 7064631a512610b52250a4d3ff1bd81551d6d1431c4eb7b72e734df6c74f4262
diff --git a/packaging/nix/umu-launcher.nix b/packaging/nix/umu-launcher.nix
@@ -16,6 +16,7 @@ python3Packages.buildPythonPackage {
   propagatedBuildInputs = [
     pyth1
     pkgs.python3Packages.xlib
+    pkgs.python3Packages.filelock
   ];
   makeFlags = [ "PYTHON_INTERPRETER=${pyth1}/bin/python" "SHELL_INTERPRETER=/run/current-system/sw/bin/bash" "DESTDIR=${placeholder "out"}" ];
   dontUseMesonConfigure = true;

diff --git a/packaging/rpm/umu-launcher.spec b/packaging/rpm/umu-launcher.spec
@@ -31,6 +31,7 @@ BuildRequires:  python3
 Requires:	python
 Requires:	python3
 Requires:	python3-xlib
+Requires:	python3-filelock
 
 
 %description

diff --git a/packaging/snap/snap/snapcraft.yaml b/packaging/snap/snap/snapcraft.yaml
@@ -247,6 +247,7 @@ parts:
     plugin: nil
     stage-packages:
       - python3-xlib
+      - python3-filelock
     prime:
       - usr/lib/python3
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -36,7 +36,10 @@ classifiers = [
   "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 urls = { repository = "https://github.com/Open-Wine-Components/umu-launcher" }
-dependencies = ["python-xlib>=0.33"]
+dependencies = [
+  "python-xlib>=0.33",
+  "filelock>=3.15.4",
+]
 
 [project.scripts]
 umu-run = "umu.__main__:main"
@@ -77,6 +80,8 @@ exclude = [
   'umu/umu_test.py',
   'umu/umu_test_plugins.py',
   'subprojects',
+  'builddir',
+  'dist',
 ]
 
 [tool.ruff]

diff --git a/requirements.in b/requirements.in
@@ -1 +1,2 @@
 python-xlib>=0.33
+filelock>=3.15.4 
diff --git a/umu/umu_proton.py b/umu/umu_proton.py
@@ -14,7 +14,9 @@
 from urllib.error import URLError
 from urllib.request import Request, urlopen
 
-from umu.umu_consts import STEAM_COMPAT
+from filelock import FileLock
+
+from umu.umu_consts import STEAM_COMPAT, UMU_LOCAL
 from umu.umu_log import log
 from umu.umu_util import run_zenity
 
@@ -301,6 +303,7 @@ def _get_latest(
     proton: str
     # Name of the Proton version, which is either UMU-Proton or GE-Proton
     version: str
+    lock: FileLock
 
     if not assets:
         return None
@@ -324,7 +327,15 @@ def _get_latest(
 
     # Use the latest UMU/GE-Proton
     try:
+        lock = FileLock(f"{UMU_LOCAL}/compatibilitytools.d.lock")
+        log.debug("Acquiring file lock '%s'...", lock.lock_file)
+        lock.acquire()
+
+        if steam_compat.joinpath(proton).is_dir():
+            raise FileExistsError
+
         _fetch_proton(env, tmp, assets)
+
         if version == "UMU-Proton":
             protons: list[Path] = [
                 file
@@ -339,11 +350,6 @@ def _get_latest(
             future.result()
         else:
             _extract_dir(tmp.joinpath(tarball), steam_compat)
-        os.environ["PROTONPATH"] = str(steam_compat.joinpath(proton))
-        env["PROTONPATH"] = os.environ["PROTONPATH"]
-        log.debug("Removing: %s", tarball)
-        thread_pool.submit(tmp.joinpath(tarball).unlink, True)
-        log.console(f"Using {version} ({proton})")
     except ValueError as e:  # Digest mismatched
         log.exception(e)
         # Since we do not want the user to use a suspect file, delete it
@@ -356,6 +362,17 @@ def _get_latest(
     except HTTPException as e:  # Download failed
         log.exception(e)
         return None
+    except FileExistsError:
+        pass
+    finally:
+        log.debug("Released file lock '%s'", lock.lock_file)
+        lock.release()
+
+    os.environ["PROTONPATH"] = str(steam_compat.joinpath(proton))
+    env["PROTONPATH"] = os.environ["PROTONPATH"]
+    log.debug("Removing: %s", tarball)
+    thread_pool.submit(tmp.joinpath(tarball).unlink, True)
+    log.console(f"Using {version} ({proton})")
 
     return env
 

diff --git a/umu/umu_runtime.py b/umu/umu_runtime.py
@@ -12,6 +12,8 @@
 from tempfile import mkdtemp
 from typing import Any
 
+from filelock import FileLock
+
 from umu.umu_consts import CONFIG, UMU_LOCAL
 from umu.umu_log import log
 from umu.umu_util import find_obsolete, run_zenity
@@ -181,7 +183,7 @@ def setup_umu(
             "Setting up Unified Launcher for Windows Games on Linux..."
         )
         local.mkdir(parents=True, exist_ok=True)
-        _install_umu(json, thread_pool)
+        _restore_umu(json, thread_pool)
         return
 
     if os.environ.get("UMU_RUNTIME_UPDATE") == "0":
@@ -219,7 +221,7 @@ def _update_umu(
     except ValueError:
         log.warning("Runtime Platform not found")
         log.console("Restoring Runtime Platform...")
-        _install_umu(json, thread_pool)
+        _restore_umu(json, thread_pool)
         return
 
     log.debug("Runtime: %s", runtime.name)
@@ -228,7 +230,7 @@ def _update_umu(
     if not local.joinpath("pressure-vessel").is_dir():
         log.warning("Runtime Platform not found")
         log.console("Restoring Runtime Platform...")
-        _install_umu(json, thread_pool)
+        _restore_umu(json, thread_pool)
         return
 
     # Restore VERSIONS.txt
@@ -244,7 +246,7 @@ def _update_umu(
         if not release.is_file():
             log.warning("Runtime Platform corrupt")
             log.console("Restoring Runtime Platform...")
-            _install_umu(json, thread_pool)
+            _restore_umu(json, thread_pool)
             return
 
         # Get the BUILD_ID value in os-release
@@ -301,10 +303,11 @@ def _update_umu(
         != sha256(local.joinpath("VERSIONS.txt").read_bytes()).digest()
     ):
         log.console("Updating steamrt to latest...")
-        _install_umu(json, thread_pool)
+        _restore_umu(json, thread_pool)
         log.debug("Removing: %s", runtime)
         rmtree(str(runtime))
         return
+
     log.console("steamrt is up to date")
 
     client_session.close()
@@ -418,3 +421,25 @@ def check_runtime(src: Path, json: dict[str, Any]) -> int:
     log.console(f"{runtime.name}: mtree is OK")
 
     return ret
+
+
+def _restore_umu(
+    json: dict[str, Any], thread_pool: ThreadPoolExecutor
+) -> None:
+    lock: FileLock = FileLock(f"{UMU_LOCAL}/umu.lock")
+
+    try:
+        log.debug("Acquiring file lock '%s'...", lock.lock_file)
+        lock.acquire()
+
+        if UMU_LOCAL.joinpath("umu").is_file():
+            raise FileExistsError
+
+        _install_umu(json, thread_pool)
+    except FileExistsError:
+        pass
+    except BaseException:
+        raise
+    finally:
+        log.debug("Released file lock '%s'", lock.lock_file)
+        lock.release()