Cleanup download info

minigalaxy/api.py

@@ -1,13 +1,17 @@
 import http
+import logging
 import os
 import time
+from typing import List
 from urllib.parse import urlencode
 import requests
 import xml.etree.ElementTree as ET
 
 from requests import Session
 
-from minigalaxy.file_info import FileInfo
+from minigalaxy.entity.download_info import DownloadInfo
+from minigalaxy.entity.game_download_info import GameDownloadInfo
+from minigalaxy.entity.xml_exception import XmlException
 from minigalaxy.game import Game
 from minigalaxy.constants import IGNORE_GAME_IDS
 from minigalaxy.config import Config
@@ -141,11 +145,11 @@ def get_info(self, game: Game) -> dict:
         return response
 
     # This returns a unique download url and a link to the checksum of the download
-    def get_download_info(self, game: Game, operating_system="linux", dlc_installers="") -> dict:
+    def get_download_info(self, game: Game, operating_system="linux", dlc_installers="") -> GameDownloadInfo:
         if dlc_installers:
             installers = dlc_installers
         else:
-            response = self.get_info(game)
+            response = self.__request("https://api.gog.com/products/{}?locale=en-US&expand=downloads".format(str(game.id)))
             installers = response["downloads"]["installers"]
         possible_downloads = []
         for installer in installers:
@@ -160,60 +164,65 @@ def get_download_info(self, game: Game, operating_system="linux", dlc_installers
         download_info = possible_downloads[0]
         for installer in possible_downloads:
             if installer['language'] == self.config.lang:
-                download_info = installer
+                download_info = GameDownloadInfo.from_dict(data=installer)
+                for download_file in installer["files"]:
+                    download_info.files.append(self.get_download_file_info(download_file["downlink"]))
                 break
             if installer['language'] == "en":
-                download_info = installer
+                download_info = GameDownloadInfo.from_dict(data=installer)
+                for download_file in installer["files"]:
+                    download_info.files.append(self.get_download_file_info(download_file["downlink"]))
 
-        # Return last entry in possible_downloads. This will either be English or the first langauge in the list
-        # This is just a backup, if the preferred language has been found, this part won't execute
         return download_info
 
-    def get_real_download_link(self, url):
-        return self.__request(url)['downlink']
-
     def get_download_file_info(self, url):
         """
         Returns some information about a downloadable file based on an XML file offered by GOG
         :param url: Url to get download and checksum links from the API
         :return: a FileInfo object with md5 set to the md5 or and empty string and size set to the file size or 0
         """
-        file_info = FileInfo(md5="", size=0)
+        checksum_data = self.__request(url)
+        print(checksum_data)
+        download_url = checksum_data["downlink"]
         try:
-            checksum_data = self.__request(url)
-            if 'checksum' in checksum_data.keys() and len(checksum_data['checksum']) > 0:
-                xml_data = self.__get_xml_checksum(checksum_data['checksum'])
-                if "md5" in xml_data.keys() and len(xml_data["md5"]) > 0:
-                    file_info.md5 = xml_data["md5"]
-                if "total_size" in xml_data.keys() and len(xml_data["total_size"]) > 0:
-                    file_info.size = int(xml_data["total_size"])
+            xml_data = self.session.get(checksum_data['checksum'])
+            if xml_data.status_code == http.HTTPStatus.OK and len(xml_data.text) > 0:
+                return DownloadInfo.from_xml(download_url=download_url, xml=xml_data.text)
+            else:
+                raise XmlException("Couldn't read xml data. Response with code {} received with the following content: {}".format(
+                    xml_data.status_code, xml_data.text
+                ))
         except requests.exceptions.RequestException as e:
-            print("Couldn't retrieve file info. Encountered HTTP exception: {}".format(e))
+            raise XmlException("Couldn't read xml data. Received RequestException") from e
 
-        if not file_info.md5:
-            print("Couldn't find md5 in xml checksum data")
+    def get_download_urls(self, game:Game, operating_system="linux", dlc_installers="") -> List[str]:
+        urls = []
+        if dlc_installers:
+            installers = dlc_installers
+        else:
+            response = self.__request("https://api.gog.com/products/{}?locale=en-US&expand=downloads".format(str(game.id)))
+            installers = response["downloads"]["installers"]
 
-        if not file_info.size:
-            print("Couldn't find file size in xml checksum data")
+        possible_downloads = []
+        for installer in installers:
+            if installer["os"] == operating_system:
+                possible_downloads.append(installer)
+        if not possible_downloads:
+            if operating_system == "linux":
+                return self.get_download_urls(game, "windows")
+            else:
+                raise NoDownloadLinkFound("Error: {} with id {} couldn't be installed".format(game.name, game.id))
 
-        return file_info
+        for installer in possible_downloads:
+            if installer['language'] == self.config.lang:
+                for download_file in installer["files"]:
+                    urls.append(download_file["downlink"])
+                break
+            if installer['language'] == "en":
+                for download_file in installer["files"]:
+                    urls.append(download_file["downlink"])
 
-    def __get_xml_checksum(self, url):
-        result = {}
-        try:
-            response = self.session.get(url)
-            if response.status_code == http.HTTPStatus.OK and len(response.text) > 0:
-                response_object = ET.fromstring(response.text)
-                if response_object and response_object.attrib:
-                    result = response_object.attrib
-            else:
-                print("Couldn't read xml data. Response with code {} received with the following content: {}".format(
-                    response.status_code, response.text
-                ))
-        except requests.exceptions.RequestException as e:
-            print("Couldn't read xml data. Received RequestException : {}".format(e))
-        finally:
-            return result
+        return urls
 
     def get_user_info(self) -> str:
         username = self.config.username

minigalaxy/download_manager.py

@@ -21,6 +21,7 @@
 import time
 import threading
 import queue
+from typing import Tuple, Dict, List
 
 from requests import Session
 from requests.exceptions import RequestException
@@ -35,15 +36,15 @@ class QueuedDownloadItem:
     """
     Wrap downloads in a simple class so we can manage when to download them
     """
-    def __init__(self, download, priority=1):
+    def __init__(self, download: Download, priority: int=1):
         """
         Create a QueuedDownloadItem with a given download and priority level
         """
         self.priority = priority
         self.item = download
         self.queue_time = time.time()
 
-    def __lt__(self, other):
+    def __lt__(self, other) -> bool:
         """
         Only compare QueuedDownloadItems on their priority level and
         time added to the queue.
@@ -98,7 +99,7 @@ def __init__(self, session: Session):
                 download_thread.start()
                 self.workers.append(download_thread)
 
-    def download(self, download):
+    def download(self, download: Download) -> None:
         """
         Add a download or list of downloads to the queue for downloading
         You can download a single Download or a list of Downloads
@@ -113,8 +114,8 @@ def download(self, download):
             for d in download:
                 self.put_in_proper_queue(d)
 
-    def put_in_proper_queue(self, download):
-        "Put the download in the proper queue"
+    def put_in_proper_queue(self, download: Download) -> None:
+        """Put the download in the proper queue"""
         # Add game type downloads to the game queue
         if download.download_type == DownloadType.GAME:
             self.__game_queue.put(QueuedDownloadItem(download, 1))
@@ -130,7 +131,7 @@ def put_in_proper_queue(self, download):
             # Add other items to the UI queue
             self.__ui_queue.put(QueuedDownloadItem(download, 0))
 
-    def download_now(self, download):
+    def download_now(self, download: Download) -> None:
         """
         Download an item with a higher priority
         Any item with the download_now priority set will get downloaded
@@ -144,7 +145,7 @@ def download_now(self, download):
         """
         self.__ui_queue.put(QueuedDownloadItem(download, 0))
 
-    def cancel_download(self, downloads):
+    def cancel_download(self, downloads: List[Download]) -> None:
         """
         Cancel a download or a list of downloads
 
@@ -167,7 +168,7 @@ def cancel_download(self, downloads):
         # cancel list
         self.cancel_queued_downloads(download_dict)
 
-    def cancel_active_downloads(self, download_dict):
+    def cancel_active_downloads(self, download_dict: Dict) -> None:
         """
         Cancel active downloads
         This is called by cancel_download
@@ -184,7 +185,7 @@ def cancel_active_downloads(self, download_dict):
                     # Remove it from the downloads to cancel
                     del download_dict[download]
 
-    def cancel_queued_downloads(self, download_dict):
+    def cancel_queued_downloads(self, download_dict: Dict) -> None:
         """
         Cancel selected downloads in the queue
         This is called by cancel_download
@@ -219,7 +220,7 @@ def cancel_queued_downloads(self, download_dict):
                     item = new_queue.get()
                     download_queue.put(item)
 
-    def cancel_current_downloads(self):
+    def cancel_current_downloads(self) -> None:
         """
         Cancel the currentl downloads
         """
@@ -228,7 +229,7 @@ def cancel_current_downloads(self):
             for download in self.active_downloads:
                 self.__cancel[download] = True
 
-    def cancel_all_downloads(self):
+    def cancel_all_downloads(self) -> None:
         """
         Cancel all current downloads queued
         """
@@ -244,16 +245,16 @@ def cancel_all_downloads(self):
 
         self.cancel_current_downloads()
 
-    def __remove_download_from_active_downloads(self, download):
-        "Remove a download from the list of active downloads"
+    def __remove_download_from_active_downloads(self, download: Download) -> None:
+        """Remove a download from the list of active downloads"""
         with self.active_downloads_lock:
             if download in self.active_downloads:
                 self.logger.debug("Removing download from active downloads list")
                 del self.active_downloads[download]
             else:
                 self.logger.debug("Didn't find download in active downloads list")
 
-    def __download_thread(self, download_queue):
+    def __download_thread(self, download_queue: queue.PriorityQueue) -> None:
         """
         The main DownloadManager thread calls this when it is created
         It checks the queue, starting new downloads when they are available
@@ -265,13 +266,13 @@ def __download_thread(self, download_queue):
                 with self.active_downloads_lock:
                     download = download_queue.get().item
                     self.active_downloads[download] = download
-                self.__download_file(download, download_queue)
+                self.__download_file(download)
                 # Mark the task as done to keep counts correct so
                 # we can use join() or other functions later
                 download_queue.task_done()
             time.sleep(0.01)
 
-    def __download_file(self, download, download_queue):
+    def __download_file(self, download: Download) -> None:
         """
         This is called by __download_thread to download a file
         It is also called directly by the thread created in download_now
@@ -309,7 +310,7 @@ def __download_file(self, download, download_queue):
             # We may want to unset current_downloads here
             # For example, if a download was added that is impossible to complete
 
-    def __prepare_location(self, save_location):
+    def __prepare_location(self, save_location: str) -> None:
         """
         Make sure the download directory exists and the file doesn't already exist
 
@@ -326,7 +327,7 @@ def __prepare_location(self, save_location):
             shutil.rmtree(save_location)
             self.logger.debug("{} is a directory. Will remove it, to make place for installer.".format(save_location))
 
-    def __get_start_point_and_download_mode(self, download):
+    def __get_start_point_and_download_mode(self, download) -> Tuple[int, str]:
         """
         Resume the previous download if possible
 
@@ -344,7 +345,7 @@ def __get_start_point_and_download_mode(self, download):
                 os.remove(download.save_location)
         return start_point, download_mode
 
-    def __download_operation(self, download, start_point, download_mode):
+    def __download_operation(self, download: Download, start_point: int, download_mode: str) -> bool:
         """
         Download the file
         This is called by __download_file to actually perform the download
@@ -384,7 +385,7 @@ def __download_operation(self, download, start_point, download_mode):
         self.logger.debug("Returning result from _download_operation: {}".format(result))
         return result
 
-    def __is_same_download_as_before(self, download):
+    def __is_same_download_as_before(self, download: Download) -> bool:
         """
         Return true if the download is the same as an item with the same save_location
         already downloaded.

minigalaxy/entity/download_chunk.py

@@ -0,0 +1,13 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class DownloadChunk:
+    chunk_id: int
+    from_byte: int
+    to_byte: int
+    method: str
+    checksum: str
+
+    def get_size(self) -> int:
+        return self.to_byte - (self.from_byte - 1)  # the -1 is because the from-byte is included

minigalaxy/entity/download_info.py

@@ -0,0 +1,57 @@
+import datetime
+from dataclasses import dataclass
+from typing import List
+
+import xml.etree.ElementTree as ET
+
+from minigalaxy.entity.download_chunk import DownloadChunk
+from minigalaxy.entity.xml_exception import XmlException
+
+
+@dataclass
+class DownloadInfo:
+    name: str
+    download_url: str
+    available: int
+    not_availablemsg: str
+    md5: str
+    chunks: List[DownloadChunk]
+    timestamp: datetime.datetime
+    total_size: int
+
+    @staticmethod
+    def from_xml(download_url: str, xml: str) -> 'DownloadInfo':
+        """
+        Convert xml data into a DownloadInfo object
+        Can throw XmlException if the xml is not readable
+        :param download_url: where to download the file
+        :param xml: xml content as string
+        :return: a new DownloadInfo object
+        """
+        try:
+            xml_data = ET.fromstring(xml)
+            if xml_data and xml_data.attrib:
+                    download_data = DownloadInfo(
+                        name=xml_data.attrib["name"],
+                        download_url=download_url,
+                        available=int(xml_data.attrib.get("available", "0")),
+                        not_availablemsg=xml_data.get("not_availablemsg", ""),
+                        md5=xml_data.attrib["md5"],
+                        chunks=[],
+                        timestamp=datetime.datetime.fromisoformat(xml_data.attrib["timestamp"]),
+                        total_size=int(xml_data.attrib["total_size"]),
+                    )
+                    for chunk in xml_data:
+                        download_data.chunks.append(
+                            DownloadChunk(
+                                chunk_id=int(chunk.attrib["id"]),
+                                from_byte=int(chunk.attrib["from"]),
+                                to_byte=int(chunk.attrib["to"]),
+                                method=chunk.attrib["method"],
+                                checksum=chunk.text,
+                            )
+                        )
+                    return download_data
+        except (KeyError, ValueError, AssertionError, ET.ParseError) as e:
+            raise XmlException(f"Could not process the following data as a download xml: {xml}") from e
+        raise XmlException(f"Could not process the following data as a download xml: {xml}")