Improve the handling of strings and unicode (#188)

* Remove a couple of unicode references

* Use six library for handling text

Also cleaned up the import section to pass isort, and changed a few
`type()` calls to use `isinstance`.

* Improve the handling of unicode

* Improve the handling of unicode

* Improve a few of the tests and use unicode

* Improve the unicode handling

* Fix the string type checks

* Fix up the unicode handling

* Make the description unicode, just to complicate things.

* Handle unicode for both python 2 and 3

* Fix slots

* Make sure the uncaching of modules works in py3

* Make sure tox installs base requirements in the venv

* Fix a couple more unicode pieces

pyangbind/lib/pybindJSON.py

@@ -19,16 +19,13 @@
 """
 from __future__ import unicode_literals
 
+import copy
+import json
 from collections import OrderedDict
 
-from pyangbind.lib.serialise import pybindJSONEncoder, pybindJSONDecoder, pybindJSONIOError
-from pyangbind.lib.serialise import pybindIETFJSONEncoder
-import json
-import copy
 import six
 
-if six.PY3:
-  unicode = str
+from pyangbind.lib.serialise import pybindIETFJSONEncoder, pybindJSONDecoder, pybindJSONEncoder, pybindJSONIOError
 
 
 def remove_path(tree, path):
@@ -55,7 +52,7 @@ def loads(d, parent_pymod, yang_base, path_helper=None, extmethods=None,
   # that this really expected a dict, so this check simply makes sure
   # that if the user really did give us a string, we're happy with that
   # without breaking other code.
-  if isinstance(d, unicode) or isinstance(d, str):
+  if isinstance(d, six.string_types + (six.text_type,)):
     d = json.loads(d, object_pairs_hook=OrderedDict)
   return pybindJSONDecoder.load_json(d, parent_pymod, yang_base,
           path_helper=path_helper, extmethods=extmethods, overwrite=overwrite)
@@ -64,7 +61,7 @@ def loads(d, parent_pymod, yang_base, path_helper=None, extmethods=None,
 def loads_ietf(d, parent_pymod, yang_base, path_helper=None,
                 extmethods=None, overwrite=False):
   # Same as above, to allow for load_ietf to work the same way
-  if isinstance(d, unicode) or isinstance(d, str):
+  if isinstance(d, six.string_types + (six.text_type,)):
     d = json.loads(d, object_pairs_hook=OrderedDict)
   return pybindJSONDecoder.load_ietf_json(d, parent_pymod, yang_base,
           path_helper=path_helper, extmethods=extmethods, overwrite=overwrite)
@@ -141,14 +138,13 @@ def lookup_subdict(dictionary, key):
     for t in tree:
       keep = True
       for k, v in select.iteritems():
+        v = six.text_type(v)
         if mode == 'default' or isinstance(tree, dict):
-          if keep and not \
-                unicode(lookup_subdict(tree[t], k.split("."))) == unicode(v):
+          if (keep and not six.text_type(lookup_subdict(tree[t], k.split("."))) == v):
             keep = False
         else:
           # handle ietf case where we have a list and might have namespaces
-          if keep and not \
-                unicode(lookup_subdict(t, k.split("."))) == unicode(v):
+          if (keep and not six.text_type(lookup_subdict(t, k.split("."))) == v):
             keep = False
       if not keep:
         key_del.append(t)

pyangbind/lib/serialise.py

@@ -27,8 +27,11 @@
 import json
 from collections import OrderedDict
 from decimal import Decimal
+
+import six
 from enum import IntEnum
-from pyangbind.lib.yangtypes import safe_name, YANGBool
+
+from pyangbind.lib.yangtypes import YANGBool, safe_name
 
 
 class WithDefaults(IntEnum):
@@ -96,10 +99,9 @@ def default(self, obj, mode='default'):
 
     # Map based on YANG type
     if orig_yangt in ["leafref"]:
-      return self.default(obj._get()) if hasattr(obj, "_get") \
-                                                  else unicode(obj)
+      return self.default(obj._get()) if hasattr(obj, "_get") else six.text_type(obj)
     elif orig_yangt in ["int64", "uint64"]:
-      return unicode(obj) if mode == "ietf" else int(obj)
+      return six.text_type(obj) if mode == "ietf" else int(obj)
     elif orig_yangt in ["identityref"]:
       if mode == "ietf":
         try:
@@ -108,20 +110,20 @@ def default(self, obj, mode='default'):
             return "%s:%s" % (obj._enumeration_dict[obj]["@module"], obj)
         except KeyError:
           pass
-        return unicode(obj)
+        return six.text_type(obj)
     elif orig_yangt in ["int8", "int16", "int32", "uint8", "uint16", "uint32"]:
       return int(obj)
     elif orig_yangt in ["int64" "uint64"]:
       if mode == "ietf":
-        return unicode(obj)
+        return six.text_type(obj)
       else:
         return int(obj)
     elif orig_yangt in ["string", "enumeration"]:
-      return unicode(obj)
+      return six.text_type(obj)
     elif orig_yangt in ["binary"]:
       return obj.to01()
     elif orig_yangt in ["decimal64"]:
-      return unicode(obj) if mode == "ietf" else float(obj)
+      return six.text_type(obj) if mode == "ietf" else float(obj)
     elif orig_yangt in ["bool"]:
       return True if obj else False
     elif orig_yangt in ["empty"]:
@@ -149,14 +151,14 @@ def default(self, obj, mode='default'):
       for k, v in obj.iteritems():
         ndict[k] = self.default(v, mode=mode)
       return ndict
-    elif type(obj) in [str, unicode]:
-      return unicode(obj)
-    elif type(obj) in [int, long]:
+    elif isinstance(obj, six.string_types + (six.text_type,)):
+      return six.text_type(obj)
+    elif isinstance(obj, six.integer_types):
       return int(obj)
-    elif type(obj) in [YANGBool, bool]:
+    elif isinstance(obj, (YANGBool, bool)):
       return bool(obj)
-    elif type(obj) in [Decimal]:
-      return unicode(obj) if mode == "ietf" else float(obj)
+    elif isinstance(obj, Decimal):
+      return six.text_type(obj) if mode == "ietf" else float(obj)
 
     raise AttributeError("Unmapped type: %s, %s, %s, %s, %s, %s" %
                                   (elem_name, orig_yangt, pybc, pyc,
@@ -171,12 +173,12 @@ def map_pyangbind_type(self, map_val, original_yang_type, obj, mode):
       return self.default(obj._get(), mode=mode)
     elif map_val in ["pyangbind.lib.yangtypes.RestrictedPrecisionDecimal", "RestrictedPrecisionDecimal"]:
       if mode == "ietf":
-        return unicode(obj)
+        return six.text_type(obj)
       return float(obj)
     elif map_val in ["bitarray.bitarray"]:
       return obj.to01()
     elif map_val in ["unicode"]:
-      return unicode(obj)
+      return six.text_type(obj)
     elif map_val in ["pyangbind.lib.yangtypes.YANGBool"]:
       if original_yang_type == "empty" and mode == "ietf":
         if obj:
@@ -193,12 +195,12 @@ def map_pyangbind_type(self, map_val, original_yang_type, obj, mode):
     elif map_val in ["long"]:
       int_size = getattr(obj, "_restricted_int_size", None)
       if mode == "ietf" and int_size == 64:
-        return unicode(obj)
+        return six.text_type(obj)
       return int(obj)
     elif map_val in ["container"]:
       return self._preprocess_element(obj.get(), mode=mode)
     elif map_val in ["decimal.Decimal"]:
-      return unicode(obj) if mode == "ietf" else float(obj)
+      return six.text_type(obj) if mode == "ietf" else float(obj)
 
 
 class pybindJSONDecoder(object):
@@ -333,7 +335,7 @@ def load_json(d, parent, yang_base, obj=None, path_helper=None,
 
   @staticmethod
   def check_metadata_add(key, data, obj):
-    keys = [unicode(k) for k in data]
+    keys = [six.text_type(k) for k in data]
     if ("@" + key) in keys:
       for k, v in data["@" + key].iteritems():
         obj._add_metadata(k, v)

pyangbind/lib/xpathhelper.py

@@ -23,13 +23,17 @@
   This module maintains an XML ElementTree for the registered Python
   classes, so that XPATH can be used to lookup particular items.
 """
+from __future__ import unicode_literals
+
+import uuid
 from collections import OrderedDict
 
-from lxml import etree
 import regex
-import uuid
-from .yangtypes import safe_name
+import six
+from lxml import etree
+
 from .base import PybindBase
+from .yangtypes import safe_name
 
 
 class YANGPathHelperException(Exception):
@@ -304,7 +308,7 @@ def _get_etree(self, object_path, caller=False):
     return retr_obj
 
   def get(self, object_path, caller=False):
-    if isinstance(object_path, str) or isinstance(object_path, unicode):
+    if isinstance(object_path, six.string_types + (six.text_type,)):
       object_path = self._path_parts(object_path)
 
     return [self._library[i.get("obj_ptr")]
@@ -323,7 +327,7 @@ def get_unique(self, object_path, caller=False,
 
   def get_list(self, object_path, caller=False,
                 exception_to_raise=YANGPathHelperException):
-    if isinstance(object_path, str) or isinstance(object_path, unicode):
+    if isinstance(object_path, six.string_types + (six.text_type,)):
       object_path = self._path_parts(object_path)
 
     parent_obj = self.get_unique(object_path[:-1], caller=caller,

pyangbind/lib/yangtypes.py

@@ -21,18 +21,15 @@
 """
 from __future__ import unicode_literals
 
-from decimal import Decimal
-from bitarray import bitarray
-import uuid
-import regex
 import collections
 import copy
+import uuid
+from decimal import Decimal
+
+import regex
 import six
+from bitarray import bitarray
 
-# For Python3
-if six.PY3:
-  unicode = str
-  basestring = str
 # Words that could turn up in YANG definition files that are actually
 # reserved names in Python, such as being builtin types. This list is
 # not complete, but will probably continue to grow.
@@ -130,7 +127,7 @@ def RestrictedClassType(*args, **kwargs):
     type of restriction placed on the class, and the restriction_arg gives
     any input data that this function needs.
   """
-  base_type = kwargs.pop("base_type", unicode)
+  base_type = kwargs.pop("base_type", six.text_type)
   restriction_type = kwargs.pop("restriction_type", None)
   restriction_arg = kwargs.pop("restriction_arg", None)
   restriction_dict = kwargs.pop("restriction_dict", None)
@@ -140,7 +137,7 @@ def RestrictedClassType(*args, **kwargs):
   # it must be a list since a restricted class can encapsulate a restricted
   # class
   current_restricted_class_type = regex.sub("<(type|class) '(?P<class>.*)'>",
-                                          "\g<class>", str(base_type))
+                                          "\g<class>", six.text_type(base_type))
   if hasattr(base_type, "_restricted_class_base"):
     restricted_class_hint = getattr(base_type, "_restricted_class_base")
     restricted_class_hint.append(current_restricted_class_type)
@@ -252,15 +249,15 @@ def range_check(value):
 
       def match_pattern_check(regexp):
         def mp_check(value):
-          if not isinstance(value, basestring):
+          if not isinstance(value, six.string_types + (six.text_type,)):
             return False
           if regex.match(convert_regexp(regexp), value):
             return True
           return False
         return mp_check
 
       def in_dictionary_check(dictionary):
-        return lambda i: unicode(i) in dictionary
+        return lambda i: six.text_type(i) in dictionary
 
       val = False
       try:
@@ -365,7 +362,7 @@ def TypedListType(*args, **kwargs):
     certain types (specified by allowed_type kwarg to the function)
     can be added to the list.
   """
-  allowed_type = kwargs.pop("allowed_type", unicode)
+  allowed_type = kwargs.pop("allowed_type", six.text_type)
   if not isinstance(allowed_type, list):
     allowed_type = [allowed_type]
 
@@ -409,11 +406,11 @@ def check(self, v):
               tmp = i(v)
               passed = True
               break
-          elif i == unicode and isinstance(v, str):
-            tmp = unicode(v)
+          elif i == six.text_type and isinstance(v, six.string_types + (six.text_type,)):
+            tmp = six.text_type(v)
             passed = True
             break
-          elif i not in [unicode, str]:
+          elif i not in six.string_types + (six.text_type,):
             # for anything other than string we try
             # and cast. Using things for string or
             # unicode gives us strange results because we get
@@ -501,7 +498,7 @@ def YANGListType(*args, **kwargs):
   extensions = kwargs.pop("extensions", None)
 
   class YANGList(object):
-    __slots__ = ('_pybind_generated_by', '_members', '_keyval',
+    __slots__ = ('_members', '_keyval',
                   '_contained_class', '_path_helper', '_yang_keys',
                   '_ordered',)
     _pybind_generated_by = "YANGListType"
@@ -584,7 +581,7 @@ def __set(self, *args, **kwargs):
         # this is a list that does not have a key specified, and hence
         # we generate a uuid that is used as the key, the method then
         # returns the uuid for the upstream process to use
-        k = str(uuid.uuid1())
+        k = six.text_type(uuid.uuid1())
 
       update = False
       if v is not None:
@@ -714,12 +711,12 @@ def _generate_key(self, *args, **kwargs):
     def _extract_key(self, obj):
       kp = self._keyval.split(" ")
       if len(kp) > 1:
-        ks = unicode()
+        ks = ''
         for k in kp:
           kv = getattr(obj, "_get_%s" % safe_name(k), None)
           if kv is None:
             raise KeyError("Invalid key attribute specified for object")
-          ks += "%s " % unicode(kv())
+          ks += "%s " % six.text_type(kv())
         return ks.rstrip(" ")
       else:
         kv = getattr(obj, "_get_%s" % safe_name(self._keyval), None)
@@ -918,7 +915,7 @@ def YANGDynClass(*args, **kwargs):
   clsslots = ['_default', '_mchanged', '_yang_name', '_choice', '_parent',
                  '_supplied_register_path', '_path_helper', '_base_type',
                  '_is_leaf', '_is_container', '_extensionsd',
-                 '_pybind_base_class', '_extmethods', '_is_keyval',
+                 '_extmethods', '_is_keyval',
                  '_register_paths', '_namespace', '_yang_type',
                  '_defining_module', '_metadata', '_is_config', '_cpresent',
                  '_presence']
@@ -1176,7 +1173,7 @@ def __init__(self, *args, **kwargs):
       self._ptr = False
       self._require_instance = require_instance
       self._type = "unicode"
-      self._utype = unicode
+      self._utype = six.text_type
 
       if len(args):
         value = args[0]
@@ -1228,15 +1225,15 @@ def __init__(self, *args, **kwargs):
             if len(path_chk) == 1 and is_yang_leaflist(path_chk[0]):
               index = 0
               for i in path_chk[0]:
-                if unicode(i) == unicode(value):
+                if six.text_type(i) == six.text_type(value):
                   found = True
                   self._referenced_object = path_chk[0][index]
                   break
                 index += 1
             else:
               found = False
               for i in path_chk:
-                if unicode(i) == unicode(value):
+                if six.text_type(i) == six.text_type(value):
                   found = True
                   self._referenced_object = i