[python3] Import six, fix urllib, stringio and dircache errors

.travis.yml

@@ -1,3 +1,10 @@
 language: python
-# command to run tests
+python:
+  - "2.7"
+  - "3.6"
+matrix:
+  allow_failures:
+    - python: "3.6"
+install:
+  - pip install -r requirements.txt
 script: nosetests

feedvalidator.py

@@ -21,6 +21,8 @@
 """
 from __future__ import print_function
 
+from six import string_types
+
 import bisect
 import codecs
 import datetime
@@ -531,7 +533,7 @@ def RunValidationOutputToFile(feed, options, output_file):
                                                options.error_types_ignore_list)
   problems = transitfeed.ProblemReporter(accumulator)
   schedule, exit_code = RunValidation(feed, options, problems)
-  if isinstance(feed, basestring):
+  if isinstance(feed, string_types):
     feed_location = feed
   else:
     feed_location = getattr(feed, 'name', repr(feed))

misc/import_ch_zurich.py

@@ -22,7 +22,7 @@
 # motivation to port it. Please see the examples directory for better
 # examples.
 
-import cStringIO
+from six import StringIO
 import csv
 import datetime
 import optparse
@@ -394,7 +394,7 @@ def Write(self, outpath):
                            ('trips.txt', self.WriteTrips),
                            ('stops.txt', self.WriteStations),
                            ('stop_times.txt', self.WriteStopTimes)]:
-      s = cStringIO.StringIO()
+      s = StringIO()
       func(s)
       out.writestr(filename, s.getvalue())
     out.close()

requirements.txt

@@ -1,2 +1,3 @@
 enum34
 pytz
+six

tests/testfeedvalidator.py

@@ -22,12 +22,10 @@
 import feedvalidator
 import os.path
 import re
-import StringIO
+from six import StringIO
 from tests import util
 import transitfeed
 import unittest
-from urllib2 import HTTPError, URLError
-import urllib2
 import zipfile
 
 
@@ -301,7 +299,7 @@ def testBadEolContext(self):
     new_zipfile_mem = self.ConvertDictToZip(content_dict)
 
     options = MockOptions()
-    output_file = StringIO.StringIO()
+    output_file = StringIO()
     feedvalidator.RunValidationOutputToFile(
         new_zipfile_mem, options, output_file)
     self.assertMatchesRegex(filename, output_file.getvalue())

tests/testkmlparser.py

@@ -20,7 +20,7 @@
 import kmlparser
 import os.path
 import shutil
-from StringIO import StringIO
+from six import StringIO
 from tests import util
 import transitfeed
 import unittest

tests/testkmlwriter.py

@@ -18,7 +18,7 @@
 
 from __future__ import absolute_import
 import os
-import StringIO
+from six import StringIO
 import tempfile
 import unittest
 import kmlparser
@@ -54,7 +54,7 @@ def _ElementToString(root):
   Returns:
     The XML string.
   """
-  output = StringIO.StringIO()
+  output = StringIO()
   ET.ElementTree(root).write(output, 'utf-8')
   return output.getvalue()
 

tests/testmerge.py

@@ -23,7 +23,7 @@
 import merge
 import os.path
 import re
-import StringIO
+from six import StringIO
 from tests import util
 import transitfeed
 import unittest
@@ -1463,7 +1463,7 @@ def testGeneratesSomeHTML(self):
     self.problem_reporter.SameIdButNotMerged(self.dataset_merger,
                                              'test', 'unknown reason')
 
-    output_file = StringIO.StringIO()
+    output_file = StringIO()
     old_feed_path = '/path/to/old/feed'
     new_feed_path = '/path/to/new/feed'
     merged_feed_path = '/path/to/merged/feed'
@@ -1479,7 +1479,7 @@ def testGeneratesSomeHTML(self):
 class MergeInSubprocessTestCase(util.TempDirTestCaseBase):
   def CopyAndModifyTestData(self, zip_path, modify_file, old, new):
     """Return path of zip_path copy with old replaced by new in modify_file."""
-    zipfile_mem = StringIO.StringIO(open(zip_path, 'rb').read())
+    zipfile_mem = StringIO(open(zip_path, 'rb').read())
     old_zip = zipfile.ZipFile(zipfile_mem, 'r')
 
     content_dict = self.ConvertZipToDict(old_zip)

tests/transitfeed/testloader.py

@@ -16,7 +16,7 @@
 from __future__ import absolute_import
 
 import re
-from StringIO import StringIO
+from six import StringIO
 import tempfile
 from tests import util
 import transitfeed

tests/transitfeed/testroute.py

@@ -17,7 +17,7 @@
 # Unit tests for the route module.
 from __future__ import absolute_import
 
-from StringIO import StringIO
+from six import StringIO
 from tests import util
 import transitfeed
 

tests/transitfeed/teststop.py

@@ -15,6 +15,8 @@
 # Unit tests for the stop module.
 from __future__ import absolute_import
 
+from six import string_types
+
 from tests import util
 import transitfeed
 
@@ -462,9 +464,9 @@ def testWithoutSchedule(self):
     stop.stop_lon = '40.02'
     self.assertEquals(stop.new_column, 'val')
     self.assertEquals(stop['new_column'], 'val')
-    self.assertTrue(isinstance(stop['stop_lat'], basestring))
+    self.assertTrue(isinstance(stop['stop_lat'], string_types))
     self.assertAlmostEqual(float(stop['stop_lat']), 5.909)
-    self.assertTrue(isinstance(stop['stop_lon'], basestring))
+    self.assertTrue(isinstance(stop['stop_lon'], string_types))
     self.assertAlmostEqual(float(stop['stop_lon']), 40.02)
     stop.Validate(self.problems)
     self.accumulator.AssertNoMoreExceptions()

tests/transitfeed/testtransfer.py

@@ -15,7 +15,7 @@
 # Unit tests for the transfer module.
 from __future__ import absolute_import
 
-from StringIO import StringIO
+from six import StringIO
 import transitfeed
 from tests import util
 

tests/transitfeed/testtrip.py

@@ -15,7 +15,7 @@
 # Unit tests for the trip module.
 from __future__ import absolute_import
 
-from StringIO import StringIO
+from six import StringIO
 from tests import util
 import transitfeed
 

tests/transitfeed/testutil.py

@@ -18,16 +18,15 @@
 
 import datetime
 import re
-import StringIO
+from six import StringIO
 import tests.util as test_util
 from transitfeed import problems
 from transitfeed.problems import ProblemReporter
 from transitfeed import stop
 from transitfeed import util
 from transitfeed import version
 import unittest
-from urllib2 import HTTPError, URLError
-import urllib2
+from six.moves import urllib
 
 
 class ColorLuminanceTestCase(test_util.TestCase):
@@ -254,14 +253,14 @@ def runTest(self):
 
 class CheckVersionTestCase(test_util.TempDirTestCaseBase):
   def setUp(self):
-    self.orig_urlopen = urllib2.urlopen
+    self.orig_urlopen = urllib.request.urlopen
     self.mock = MockURLOpen()
     self.accumulator = test_util.RecordingProblemAccumulator(self)
     self.problems = ProblemReporter(self.accumulator)
 
   def tearDown(self):
     self.mock = None
-    urllib2.urlopen = self.orig_urlopen
+    urllib.request.urlopen = self.orig_urlopen
 
   def testAssignedDifferentVersion(self):
     util.CheckVersion(self.problems, '100.100.100')
@@ -275,33 +274,33 @@ def testAssignedSameVersion(self):
     self.accumulator.AssertNoMoreExceptions()
 
   def testGetCorrectReturns(self):
-    urllib2.urlopen = self.mock.mockedConnectSuccess
+    urllib.request.urlopen = self.mock.mockedConnectSuccess
     util.CheckVersion(self.problems)
     self.accumulator.PopException('NewVersionAvailable')
 
   def testPageNotFound(self):
-    urllib2.urlopen = self.mock.mockedPageNotFound
+    urllib.request.urlopen = self.mock.mockedPageNotFound
     util.CheckVersion(self.problems)
     e = self.accumulator.PopException('OtherProblem')
     self.assertTrue(re.search(r'we failed to reach', e.description))
     self.assertTrue(re.search(r'Reason: Not Found \[404\]', e.description))
 
   def testConnectionTimeOut(self):
-    urllib2.urlopen = self.mock.mockedConnectionTimeOut
+    urllib.request.urlopen = self.mock.mockedConnectionTimeOut
     util.CheckVersion(self.problems)
     e = self.accumulator.PopException('OtherProblem')
     self.assertTrue(re.search(r'we failed to reach', e.description))
     self.assertTrue(re.search(r'Reason: Connection timed', e.description))
 
   def testGetAddrInfoFailed(self):
-    urllib2.urlopen = self.mock.mockedGetAddrInfoFailed
+    urllib.request.urlopen = self.mock.mockedGetAddrInfoFailed
     util.CheckVersion(self.problems)
     e = self.accumulator.PopException('OtherProblem')
     self.assertTrue(re.search(r'we failed to reach', e.description))
     self.assertTrue(re.search(r'Reason: Getaddrinfo failed', e.description))
 
   def testEmptyIsReturned(self):
-    urllib2.urlopen = self.mock.mockedEmptyIsReturned
+    urllib.request.urlopen = self.mock.mockedEmptyIsReturned
     util.CheckVersion(self.problems)
     e = self.accumulator.PopException('OtherProblem')
     self.assertTrue(re.search(r'we had trouble parsing', e.description))
@@ -310,20 +309,20 @@ def testEmptyIsReturned(self):
 class MockURLOpen:
   """Pretend to be a urllib2.urlopen suitable for testing."""
   def mockedConnectSuccess(self, request):
-    return StringIO.StringIO('latest_version=100.0.1')
+    return StringIO('latest_version=100.0.1')
 
   def mockedPageNotFound(self, request):
-    raise HTTPError(request.get_full_url(), 404, 'Not Found',
+    raise urllib.error.HTTPError(request.get_full_url(), 404, 'Not Found',
                     request.header_items(), None)
 
   def mockedConnectionTimeOut(self, request):
-    raise URLError('Connection timed out')
+    raise urllib.error.URLError('Connection timed out')
 
   def mockedGetAddrInfoFailed(self, request):
-    raise URLError('Getaddrinfo failed')
+    raise urllib.error.URLError('Getaddrinfo failed')
 
   def mockedEmptyIsReturned(self, request):
-    return StringIO.StringIO()
+    return StringIO()
 
 
 if __name__ == '__main__':

tests/util.py

@@ -18,11 +18,10 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
-import dircache
 import os
 import os.path
 import re
-import cStringIO as StringIO
+from six import StringIO
 import shutil
 import subprocess
 import sys
@@ -76,7 +75,7 @@ def DataPath(path):
 
 def GetDataPathContents():
   here = os.path.dirname(__file__)
-  return dircache.listdir(os.path.join(here, 'data'))
+  return os.listdir(os.path.join(here, 'data'))
 
 
 class TestCase(unittest.TestCase):
@@ -97,7 +96,7 @@ class RedirectStdOutTestCaseBase(TestCase):
   """Save stdout to the StringIO buffer self.this_stdout"""
   def setUp(self):
     self.saved_stdout = sys.stdout
-    self.this_stdout = StringIO.StringIO()
+    self.this_stdout = StringIO()
     sys.stdout = self.this_stdout
 
   def tearDown(self):
@@ -198,7 +197,7 @@ def ConvertDictToZip(self, dict):
 
     Returns:
         The new file's in-memory contents as a file-like object."""
-    zipfile_mem = StringIO.StringIO()
+    zipfile_mem = StringIO()
     zip = zipfile.ZipFile(zipfile_mem, 'a')
     for arcname, contents in dict.items():
       zip.writestr(arcname, contents)
@@ -320,7 +319,7 @@ def GetArchiveNames(self):
 
   def CreateZip(self):
     """Create an in-memory GTFS zipfile from the contents of the file dict."""
-    self.zipfile = StringIO.StringIO()
+    self.zipfile = StringIO()
     self.zip = zipfile.ZipFile(self.zipfile, 'a')
     for (arcname, contents) in self.zip_contents.items():
       self.zip.writestr(arcname, contents)

transitfeed/loader.py

@@ -16,7 +16,7 @@
 
 from __future__ import absolute_import
 import codecs
-import cStringIO as StringIO
+from six import StringIO,string_types
 import csv
 import os
 import re
@@ -74,7 +74,7 @@ def _DetermineFormat(self):
       assert not self._path
       return True
 
-    if not isinstance(self._path, basestring) and hasattr(self._path, 'read'):
+    if not isinstance(self._path, string_types) and hasattr(self._path, 'read'):
       # A file-like object, used for testing with a StringIO file
       self._zip = zipfile.ZipFile(self._path, mode='r')
       return True
@@ -152,7 +152,7 @@ def _ReadCsvDict(self, file_name, cols, required, deprecated):
     if not contents:
       return
 
-    eol_checker = util.EndOfLineChecker(StringIO.StringIO(contents),
+    eol_checker = util.EndOfLineChecker(StringIO(contents),
                                    file_name, self._problems)
     # The csv module doesn't provide a way to skip trailing space, but when I
     # checked 15/675 feeds had trailing space in a header row and 120 had spaces
@@ -287,7 +287,7 @@ def _ReadCSV(self, file_name, cols, required, deprecated):
     if not contents:
       return
 
-    eol_checker = util.EndOfLineChecker(StringIO.StringIO(contents),
+    eol_checker = util.EndOfLineChecker(StringIO(contents),
                                    file_name, self._problems)
     reader = csv.reader(eol_checker)  # Use excel dialect