Dataexample update

CHANGELOG.md

@@ -51,6 +51,7 @@
     - BlockOperator that would return a BlockDataContainer of shape (1,1) now returns the appropriate DataContainer. BlockDataContainer direct and adjoint methods accept DataContainer as parameter (#1802).
     - BlurringOperator: remove check for geometry class (old SIRF integration bug) (#1807)
     - The `ZeroFunction` and `ConstantFunction` now have a Lipschitz constant of 1. (#1768)
+    - Update dataexample remote data download to work with windows and use zenodo_get for data download (#1774)
   - Changes that break backwards compatibility:
     - Merged the files `BlockGeometry.py` and `BlockDataContainer.py` in `framework` to one file `block.py`. Please use `from cil.framework import BlockGeometry, BlockDataContainer` as before (#1799)
     - Bug fix in `FGP_TV` function to set the default behaviour not to enforce non-negativity (#1826).

Wrappers/Python/cil/utilities/dataexample.py

@@ -25,10 +25,9 @@
 import os.path
 import sys
 from zipfile import ZipFile
-from urllib.request import urlopen
-from io import BytesIO
 from scipy.io import loadmat
 from cil.io import NEXUSDataReader, NikonDataReader, ZEISSDataReader
+from zenodo_get import zenodo_get
 
 class DATA(object):
     @classmethod
@@ -46,19 +45,13 @@ def get(cls, size=None, scale=(0,1), **kwargs):
 class REMOTEDATA(DATA):
 
     FOLDER = ''
-    URL = ''
-    FILE_SIZE = ''
+    ZENODO_RECORD = ''
+    ZIP_FILE = ''
 
     @classmethod
     def get(cls, data_dir):
         return None
 
-    @classmethod
-    def _download_and_extract_from_url(cls, data_dir):
-        with urlopen(cls.URL) as response:
-            with BytesIO(response.read()) as bytes, ZipFile(bytes) as zipfile:
-                zipfile.extractall(path = data_dir)
-
     @classmethod
     def download_data(cls, data_dir):
         '''
@@ -71,14 +64,20 @@ def download_data(cls, data_dir):
 
         '''
         if os.path.isdir(os.path.join(data_dir, cls.FOLDER)):
-            print("Dataset already exists in " + data_dir)
+            print("Dataset folder already exists in " + data_dir)
         else:
-            if input("Are you sure you want to download " + cls.FILE_SIZE + " dataset from " + cls.URL + " ? (y/n)") == "y":
-                print('Downloading dataset from ' + cls.URL)
-                cls._download_and_extract_from_url(os.path.join(data_dir,cls.FOLDER))
-                print('Download complete')
+            # get user confirmation for download
+            if input("Are you sure you want to download {} dataset from Zenodo record {} ? (y/n)").format(cls.ZIP_FILE, cls.ZENODO_RECORD) == "y": 
+                zenodo_get([cls.ZENODO_RECORD, '-g', cls.ZIP_FILE, '-o', data_dir])
+
+                # unzip file
+                with ZipFile(os.path.join(data_dir, cls.ZIP_FILE), 'r') as zip_ref:
+                    zip_ref.extractall(os.path.join(data_dir, cls.FOLDER))
+                os.remove(os.path.join(data_dir, cls.ZIP_FILE))
+
             else:
                 print('Download cancelled')
+                return
 
 class BOAT(CILDATA):
     @classmethod
@@ -195,10 +194,12 @@ def get(cls, **kwargs):
 class WALNUT(REMOTEDATA):
     '''
     A microcomputed tomography dataset of a walnut from https://zenodo.org/records/4822516
+    Download the dataset using dataexample.WALNUT.download_data(data_dir)
+    Then load the dataset using dataexample.WALNUT.get(data_dir)
     '''
     FOLDER = 'walnut'
-    URL = 'https://zenodo.org/record/4822516/files/walnut.zip'
-    FILE_SIZE = '6.4 GB'
+    ZENODO_RECORD = '4822516'
+    ZIP_FILE = 'walnut.zip'
 
     @classmethod
     def get(cls, data_dir):
@@ -215,6 +216,11 @@ def get(cls, data_dir):
         -------
         ImageData
             The walnut dataset
+
+        Example
+        --------
+        dataexample.WALNUT.get(data_dir) # if the data is already in data_dir
+
         '''
         filepath = os.path.join(data_dir, cls.FOLDER, 'valnut','valnut_2014-03-21_643_28','tomo-A','valnut_tomo-A.txrm')
         try:
@@ -227,10 +233,12 @@ def get(cls, data_dir):
 class USB(REMOTEDATA):
     '''
     A microcomputed tomography dataset of a usb memory stick from https://zenodo.org/records/4822516
+    Download the dataset using dataexample.USB.download_data(data_dir)
+    Then load the dataset using dataexample.USB.get(data_dir)
     '''
-    FOLDER = 'USB'
-    URL = 'https://zenodo.org/record/4822516/files/usb.zip'
-    FILE_SIZE = '3.2 GB'
+    FOLDER = 'USB' 
+    ZENODO_RECORD = '4822516'
+    ZIP_FILE = 'usb.zip'
 
     @classmethod
     def get(cls, data_dir):
@@ -247,6 +255,10 @@ def get(cls, data_dir):
         -------
         ImageData
             The usb dataset
+
+        Example
+        --------
+        dataexample.USB.get(data_dir) # if the data is already in data_dir
         '''
         filepath = os.path.join(data_dir, cls.FOLDER, 'gruppe 4','gruppe 4_2014-03-20_1404_12','tomo-A','gruppe 4_tomo-A.txrm')
         try:
@@ -259,10 +271,12 @@ def get(cls, data_dir):
 class KORN(REMOTEDATA):
     '''
     A microcomputed tomography dataset of a sunflower seeds in a box from https://zenodo.org/records/6874123
+    Download the dataset using dataexample.KORN.download_data(data_dir)
+    Then load the dataset using dataexample.KORN.get(data_dir)
     '''
     FOLDER = 'korn'
-    URL = 'https://zenodo.org/record/6874123/files/korn.zip'
-    FILE_SIZE = '2.9 GB'
+    ZENODO_RECORD = '6874123'
+    ZIP_FILE = 'korn.zip'
 
     @classmethod
     def get(cls, data_dir):
@@ -279,6 +293,10 @@ def get(cls, data_dir):
         -------
         ImageData
             The korn dataset
+
+        Example
+        --------
+        dataexample.KORN.get(data_dir) # if the data is already in data_dir
         '''
         filepath = os.path.join(data_dir, cls.FOLDER, 'Korn i kasse','47209 testscan korn01_recon.xtekct')
         try:
@@ -293,10 +311,39 @@ class SANDSTONE(REMOTEDATA):
     '''
     A synchrotron x-ray tomography dataset of sandstone from https://zenodo.org/records/4912435
     A small subset of the data containing selected projections and 4 slices of the reconstruction
+    Download the dataset using dataexample.SANDSTONE.download_data(data_dir)
+    Then load the dataset using dataexample.SANDSTONE.get(data_dir)
     '''
     FOLDER = 'sandstone'
-    URL = 'https://zenodo.org/records/4912435/files/small.zip'
-    FILE_SIZE = '227 MB'
+    ZENODO_RECORD = '4912435'
+    ZIP_FILE = 'small.zip'
+
+    @classmethod
+    def get(cls, data_dir, filename):
+        '''
+        A synchrotron x-ray tomography dataset of sandstone from https://zenodo.org/records/4912435
+        A small subset of the data containing selected projections and 4 slices of the reconstruction
+        Parameters
+        ----------
+        data_dir: str
+           The path to the directory where the dataset is stored. Data can be downloaded with dataexample.SANDSTONE.download_data(data_dir)
+
+        file: str
+            The slices or projections to return, specify the path to the file within the data_dir
+
+        Returns
+        -------
+        ImageData
+            The selected sandstone dataset
+
+        Example
+        --------
+        dataexample.SANDSTONE.get(data_dir) # if the data is already in data_dir
+        '''
+        extension = os.path.splitext(filename)[1]
+        if extension == '.mat':
+            return loadmat(os.path.join(data_dir,filename))
+
 
 class TestData(object):
     '''Class to return test data

Wrappers/Python/test/test_dataexample.py

@@ -25,11 +25,11 @@
 from testclass import CCPiTestClass
 import platform
 import numpy as np
-from unittest.mock import patch, MagicMock 
-from urllib import request
+from unittest.mock import patch 
 from zipfile import ZipFile
 from io import StringIO
-from tempfile import NamedTemporaryFile
+import uuid
+from zenodo_get import zenodo_get
 
 initialise_tests()
 
@@ -157,116 +157,100 @@ def test_load_SIMULATED_CONE_BEAM_DATA(self):
 class TestRemoteData(unittest.TestCase):
 
     def setUp(self):
-
         self.data_list = ['WALNUT','USB','KORN','SANDSTONE']
-        self.shapes_path = os.path.join(dataexample.CILDATA.data_dir, dataexample.TestData.SHAPES)
 
-    def mock_urlopen(self, mock_urlopen, zipped_bytes):
-        mock_response = MagicMock()
-        mock_response.read.return_value = zipped_bytes
-        mock_response.__enter__.return_value = mock_response
-        mock_urlopen.return_value = mock_response
 
-    @unittest.skipIf(platform.system() == 'Windows', "Skip on Windows")   
-    @patch('cil.utilities.dataexample.urlopen')
-    def test_unzip_remote_data(self, mock_urlopen):
-        '''
-        Test the _download_and_extract_data_from_url function correctly extracts files from a byte string
-        The zipped byte string is mocked using a temporary local zip file
+    def mock_zenodo_get(*args):
+        # mock zenodo_get by making a zip file containing the shapes test data when the function is called
+        shapes_path = os.path.join(dataexample.CILDATA.data_dir, dataexample.TestData.SHAPES)
+        with ZipFile(os.path.join(args[0][4], args[0][2]), mode='w') as zip_file:
+            zip_file.write(shapes_path, arcname=dataexample.TestData.SHAPES)
+
+
+    @patch('cil.utilities.dataexample.input', return_value='y')
+    @patch('cil.utilities.dataexample.zenodo_get', side_effect=mock_zenodo_get)
+    def test_download_data_input_y(self, mock_zenodo_get, input):
         '''
-
-        # create a temporary zip file to test the function
-        with NamedTemporaryFile(suffix = '.zip') as tf:
-            tmp_path = os.path.dirname(tf.name)
-            tmp_dir = os.path.splitext(os.path.basename(tf.name))[0]
-            with ZipFile(tf.name, mode='w') as zip_file:
-                zip_file.write(self.shapes_path, arcname=dataexample.TestData.SHAPES)
-
-            with open(tf.name, 'rb') as zip_file:
-                zipped_bytes = zip_file.read()
-
-        self.mock_urlopen(mock_urlopen, zipped_bytes)
-        dataexample.REMOTEDATA._download_and_extract_from_url(os.path.join(tmp_path, tmp_dir))
+        Test the download_data function, when the user input is 'y' to 'are you sure you want to download data'
+        The user input to confirm the download is mocked as 'y'
+        The zip file download is mocked by creating a zip file locally
+        Test the download_data function correctly extracts files from the zip file
+        '''        
+        # create a temporary folder in the CIL data directory
+        tmp_dir = os.path.join(dataexample.CILDATA.data_dir, str(uuid.uuid4()))
+        os.makedirs(tmp_dir)
+        # redirect print output
+        capturedOutput = StringIO()                
+        sys.stdout = capturedOutput
+        for data in self.data_list:
+            test_func = getattr(dataexample, data)
+            test_func.download_data(tmp_dir)
+            # Test the data file exists
+            self.assertTrue(os.path.isfile(os.path.join(tmp_dir, getattr(test_func, 'FOLDER'), dataexample.TestData.SHAPES)), 
+                            msg = "Download data test failed with dataset " + data)
+            # Test the zip file is removed
+            self.assertFalse(os.path.isfile(os.path.join(tmp_dir, getattr(test_func, 'ZIP_FILE'))))
+        # return to standard print output
+        sys.stdout = sys.__stdout__
+        shutil.rmtree(tmp_dir)
 
-        self.assertTrue(os.path.isfile(os.path.join(tmp_path, tmp_dir, dataexample.TestData.SHAPES)))
 
-        if os.path.exists(os.path.join(tmp_path,tmp_dir)):
-            shutil.rmtree(os.path.join(tmp_path,tmp_dir)) 
-
-    @unittest.skipIf(platform.system() == 'Windows', "Skip on Windows")   
-    @patch('cil.utilities.dataexample.input', return_value='n')    
-    @patch('cil.utilities.dataexample.urlopen')
-    def test_download_data_input_n(self, mock_urlopen, input):
+    @patch('cil.utilities.dataexample.input', return_value='n')
+    @patch('cil.utilities.dataexample.zenodo_get', side_effect=mock_zenodo_get)   
+    def test_download_data_input_n(self, mock_zenodo_get, input):
         '''
         Test the download_data function, when the user input is 'n' to 'are you sure you want to download data'
-        The zipped byte string is mocked using a temporary local zip file
         '''
-
-        # create a temporary zip file to test the function
-        with NamedTemporaryFile(suffix = '.zip') as tf:
-            tmp_path = os.path.dirname(tf.name)
-            tmp_dir = os.path.splitext(os.path.basename(tf.name))[0]
-            with ZipFile(tf.name, mode='w') as zip_file:
-                zip_file.write(self.shapes_path, arcname=dataexample.TestData.SHAPES)
-
-            with open(tf.name, 'rb') as zip_file:
-                    zipped_bytes = zip_file.read()
-
-        self.mock_urlopen(mock_urlopen, zipped_bytes)
-
+        # create a temporary folder in the CIL data directory
+        tmp_dir = os.path.join(dataexample.CILDATA.data_dir, str(uuid.uuid4()))
+        os.makedirs(tmp_dir)
         for data in self.data_list:
             # redirect print output
-            capturedOutput = StringIO()                 
-            sys.stdout = capturedOutput 
+            capturedOutput = StringIO()
+            sys.stdout = capturedOutput
             test_func = getattr(dataexample, data)
-            test_func.download_data(os.path.join(tmp_path, tmp_dir))
-            self.assertFalse(os.path.isfile(os.path.join(tmp_path, tmp_dir, test_func.FOLDER, dataexample.TestData.SHAPES)), msg = "Failed with dataset " + data)
-            self.assertEqual(capturedOutput.getvalue(),'Download cancelled\n', msg = "Failed with dataset " + data)
+            test_func.download_data(tmp_dir)
+            self.assertFalse(os.path.isfile(os.path.join(tmp_dir, getattr(test_func, 'FOLDER'), dataexample.TestData.SHAPES)), 
+                             msg = "Download dataset test failed with dataset " + data)
+            self.assertEqual(capturedOutput.getvalue(),'Download cancelled\n', 
+                             msg = "Download dataset test failed with dataset " + data)
             # return to standard print output
             sys.stdout = sys.__stdout__ 
 
-        if os.path.exists(os.path.join(tmp_path,tmp_dir)):
-            shutil.rmtree(os.path.join(tmp_path,tmp_dir)) 
+        shutil.rmtree(tmp_dir)
+
 
-    @unittest.skipIf(platform.system() == 'Windows', "Skip on Windows")   
-    @patch('cil.utilities.dataexample.input', return_value='y')    
-    @patch('cil.utilities.dataexample.urlopen')
-    def test_download_data_input_y(self, mock_urlopen, input):
+    @patch('cil.utilities.dataexample.input', return_value='y')
+    def test_download_data_empty(self, input):
         '''
-        Test the download_data function, when the user input is 'y' to 'are you sure you want to download data'
-        The zipped byte string is mocked using a temporary local zip file
+        Test an error is raised when download_data is used on an empty Zenodo record
         '''
+        remote_data = dataexample.REMOTEDATA
+        remote_data.ZENODO_RECORD = 'empty'
+        remote_data.FOLDER = 'empty'
 
-        with NamedTemporaryFile(suffix = '.zip') as tf:
-            tmp_path = os.path.dirname(tf.name)
-            tmp_dir = os.path.splitext(os.path.basename(tf.name))[0]
-            with ZipFile(tf.name, mode='w') as zip_file:
-                zip_file.write(self.shapes_path, arcname=dataexample.TestData.SHAPES)
-
-            with open(tf.name, 'rb') as zip_file:
-                    zipped_bytes = zip_file.read()
-
-        self.mock_urlopen(mock_urlopen, zipped_bytes)
+        with self.assertRaises(ValueError):
+            remote_data.download_data('.')
 
-        # redirect print output
-        capturedOutput = StringIO()                 
-        sys.stdout = capturedOutput         
+    def test_zenodo_record_exists(self):
+        '''
+        Test all Zenodo records can be accessed and the correct zip files are found
+        '''
+        tmp_dir = os.path.join(dataexample.CILDATA.data_dir, str(uuid.uuid4()))
+        os.makedirs(tmp_dir)
+        test_file = os.path.join(tmp_dir, 'test_file.txt')
 
         for data in self.data_list:
             test_func = getattr(dataexample, data)
-            test_func.download_data(os.path.join(tmp_path, tmp_dir))
-            self.assertTrue(os.path.isfile(os.path.join(tmp_path, tmp_dir, test_func.FOLDER, dataexample.TestData.SHAPES)), msg = "Failed with dataset " + data)
-
-        # return to standard print output
-        sys.stdout = sys.__stdout__ 
-
-        if os.path.exists(os.path.join(tmp_path,tmp_dir)):
-            shutil.rmtree(os.path.join(tmp_path,tmp_dir))
-
-
-    def test_download_data_bad_URL(self):
-        '''
-        Test an error is raised when _download_and_extract_from_url has an empty URL
-        '''
-        with self.assertRaises(ValueError):
-            dataexample.REMOTEDATA._download_and_extract_from_url('.')
+            zip_file_name = 'https://zenodo.org/records/' + test_func.ZENODO_RECORD + '/files/' + test_func.ZIP_FILE
+            zenodo_get([test_func.ZENODO_RECORD, '-w', test_file])
+            with open(test_file) as f:
+                self.assertTrue(zip_file_name in f.read())
+            if os.path.exists(test_file):
+                os.remove(test_file)
+
+        shutil.rmtree(tmp_dir)
+
+    def test_a(self):
+        from cil.utilities.dataexample import WALNUT
+