Update/Fixup string encoding to for Python3 and fix UTF-8 for Python2 too

.github/workflows/main.yml

@@ -38,7 +38,6 @@ jobs:
 
           # FIXME/TODO: These checks by pylint --py3k are currently failing:
           py3_todo: "\
-            unicode-builtin,\
             comprehension-escape,\
             dict-keys-not-iterating,\
             old-division"

README.md

@@ -73,7 +73,6 @@ Join the XSConsole community by creating issues in the repository to report bugs
 - **As per this agreement**, until that point, changes supporting **only Python3.6+** must go to a **py3 feature branch**.
 - The list of `TODOs` remaining for the Python3 upgrade checks (`pylint --py3k`) to be fulfilled is at [line 40 of `.github/workflows/main.yml`](https://github.com/xenserver-next/xsconsole/blob/master/.github/workflows/main.yml#L40).
   Currently, the list of `pylint --py3k` `TODOs` is:
-  - `unicode-builtin` (a PR using a function for Py2 and Py3 is working and is coming next)
   - `comprehension-escape`
   - `dict-keys-not-iterating`
   - `old-division`

XSConsoleCurses.py

@@ -13,13 +13,19 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-import curses, sys
+import curses
+import locale
+import sys
 
 from XSConsoleBases import *
 from XSConsoleConfig import *
 from XSConsoleLang import *
 from XSConsoleState import *
 
+
+class Terminal:
+    charset_encoding = "utf-8"  # global: Charset encoding of the terminal: We encode to/from this charset
+
 class CursesPalette:
     pairIndex = 1
     colours = {}
@@ -174,10 +180,19 @@ def ClippedAddStr(self,  inString, inX,  inY,  inColour): # Internal use
             if len(clippedStr) > 0:
                 try:
                     encodedStr = clippedStr
-                    if isinstance(clippedStr, unicode):
-                        encodedStr = clippedStr.encode('utf-8')
+                    if sys.version_info >= (3, 0) and isinstance(clippedStr, str):
+                        # encode the string into bytes using the terminal's charset encoding
+                        encodedStr = clippedStr.encode(Terminal.charset_encoding)
+                    elif not isinstance(encodedStr, str):
+                        # encode the Python2 unicode string into bytes using the terminal's charset encoding:
+                        encodedStr = convert_anything_to_str(clippedStr, Terminal.charset_encoding)
                         # Clear field here since addstr will clear len(encodedStr)-len(clippedStr) too few spaces
-                        self.win.addstr(inY, xPos, len(clippedStr)*' ', CursesPalette.ColourAttr(FirstValue(inColour, self.defaultColour)))
+                        self.win.addstr(
+                            inY,
+                            xPos,
+                            " " * len(encodedStr),
+                            CursesPalette.ColourAttr(FirstValue(inColour, self.defaultColour)),
+                        )
                         self.win.refresh()
                     self.win.addstr(inY, xPos, encodedStr, CursesPalette.ColourAttr(FirstValue(inColour, self.defaultColour)))
                 except Exception as e:
@@ -277,12 +292,13 @@ def Snapshot(self):
         retVal = []
         if self.title != "":
             retVal.append(self.title)
+        # When reading bytes from the termial, decode the input bytes in the Terinal's charset encoding to str:
         if self.hasBox:
-            for i in range(1, self.ySize-1):
-                retVal.append(self.win.instr(i, 1, self.xSize-2))
+            for i in range(1, self.ySize - 1):
+                retVal.append(convert_anything_to_str(self.win.instr(i, 1, self.xSize - 2), Terminal.charset_encoding))
         else:
             for i in range(self.ySize):
-                retVal.append(self.win.instr(i, 0, self.xSize))
+                retVal.append(convert_anything_to_str(self.win.instr(i, 0, self.xSize), Terminal.charset_encoding))
 
         return retVal
 
@@ -305,9 +321,35 @@ def Delete(self):
 
 class CursesScreen(CursesPane):
     def __init__(self):
+        # Set the locale for all categories to the user's default settings (specified in the
+        # LANG and LC_* environment variables) and get the user's preferred charset encoding:
+        # Needed by python2-curses to support more of UTF-8 than just lower-case latin chars:
+
+        preferred_charset_encoding_by_user = locale.getpreferredencoding()
+        if preferred_charset_encoding_by_user != "UTF-8":
+            locale.setlocale(locale.LC_ALL, 'en_US')
+        else:
+            locale.setlocale(locale.LC_ALL, "")
 
         self.win = curses.initscr()
 
+        # XTerm (and it's the clones, that's nearly all major other terminal emuluations)
+        # follows the ISO 2022 standard for character set switching. Traditionally, xsconsole
+        # switched XTerm-compatibles unto ISO-8859-1 mode, which is not a perfect match for
+        # Xen-API names which are exclusively UTF-8. Default to UTF-8 mode, unless the users's
+        # locale charset environment is configured to use an ISO-8809-1 or -15(1 plus Euro sign):
+
+        if preferred_charset_encoding_by_user in ("ISO-8859-1", "ISO-8859-15"):
+            sys.stdout.write("\033%@")  # Put the terminal into ISO 8859-1 mode
+            Terminal.charset_encoding = preferred_charset_encoding_by_user
+        else:
+            sys.stdout.write("\033%G")  # Put the terminal into UTF-8 mode (default enconding)
+            Terminal.charset_encoding = "utf-8"
+
+        # Helpful for clarity when debugging:
+        # self.win.addstr(Terminal.charset_encoding)
+        # self.win.refresh()
+
         (ySize, xSize) = self.win.getmaxyx()
         CursesPane.__init__(self, 0, 0, xSize, ySize, 0, 0)
         curses.noecho()

XSConsoleLang.py

@@ -13,10 +13,35 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+import sys
+
+import XenAPI  # For XenAPI.Failure
+
 from XSConsoleConfig import *
 from XSConsoleLangErrors import *
 
-import XenAPI # For XenAPI.Failure
+# Assign the unicode text type to a variable for use with isinstance():
+if sys.version_info >= (3, 0):
+    text_type = str
+else:
+    text_type = unicode  # pyright:ignore[reportUndefinedVariable] # pylint: disable=unicode-builtin
+
+
+def convert_anything_to_str(arg, encoding="utf-8"):
+    """Converts anything into the native "str" type of the Python version, without u'str' or b'str'"""
+    if arg is None or isinstance(arg, str):
+        return arg  # Already str or None (checked by some callers), return it as-is:
+
+    # If "unicode" text or "bytes", en- or decode accordingly:
+    if isinstance(arg, (text_type, bytes)):
+        if sys.version_info > (3, 0):
+            # Python3: Decode UTF-8 bytes into the native Python3 Unicode string:
+            return arg.decode(encoding)
+        else:
+            return arg.encode(encoding)
+
+    # Not string-like (a number or object): Get a "str" of it using str(arg):
+    return str(arg)
 
 
 # Global function
@@ -79,19 +104,18 @@ def ToString(cls, inLabel):
         if isinstance(inLabel, XenAPI.Failure):
             retVal = cls.XapiError(inLabel.details)
             cls.LogError(retVal)
+
         elif isinstance(inLabel, Exception):
-            exn_strings = []
-            for arg in inLabel.args:
-                if isinstance(arg, unicode):
-                    exn_strings.append(arg.encode('utf-8'))
-                else:
-                    exn_strings.append(str(arg))
-            retVal = str(tuple(exn_strings))
+            # convert the args like "str(tuple)" would, with all args like: 'converted-to-string', :
+            retVal = ""
+            for exception_arg in inLabel.args:
+                retVal += ", " if retVal else ""
+                retVal += "'" + convert_anything_to_str(exception_arg) + "'"
+            retVal = "(" + retVal + ")"
             cls.LogError(retVal)
-        else:
-            if isinstance(inLabel, unicode):
-                inLabel = inLabel.encode('utf-8')
-            retVal = inLabel
+
+        else:  # convert anything else directly to a str and call the stringHook with it:
+            retVal = convert_anything_to_str(inLabel)
             if cls.stringHook is not None:
                 cls.stringHook(retVal)
         return retVal

XSConsoleTerm.py

@@ -83,7 +83,6 @@ def Enter(self):
         while not doQuit:
             try:
                 try:
-                    sys.stdout.write("\033%@") # Select default character set, ISO 8859-1 (ISO 2022)
                     if os.path.isfile("/bin/setfont"):
                         os.system("/bin/setfont") # Restore the default font
                     if '-f' in sys.argv:

plugins-base/XSFeatureSRCommon.py

@@ -211,7 +211,7 @@ def UpdateFieldsINITIAL(self):
         pane.ResetFields()
 
         sr = HotAccessor().sr[self.srHandle]
-        srName = sr.name_label(None).encode('utf-8')
+        srName = convert_anything_to_str(sr.name_label(None))
         if srName is None:
             pane.AddTitleField(Lang("The Virtual Machine is no longer present"))
         else:
@@ -226,7 +226,7 @@ def UpdateFieldsCONFIRM(self):
         pane.ResetFields()
 
         sr = HotAccessor().sr[self.srHandle]
-        srName = sr.name_label(None).encode('utf-8')
+        srName = convert_anything_to_str(sr.name_label(None))
         if srName is None:
             pane.AddTitleField(Lang("The Storage Repository is no longer present"))
         else:
@@ -282,7 +282,7 @@ def Commit(self):
         Layout.Inst().PopDialogue()
 
         operationName = SRUtils.OperationName(self.operation)
-        srName = HotAccessor().sr[self.srHandle].name_label(Lang('<Unknown>')).encode('utf-8')
+        srName = convert_anything_to_str(HotAccessor().sr[self.srHandle].name_label(Lang('<Unknown>')))
         messagePrefix = operationName + Lang(' operation on ') + srName + ' '
         Layout.Inst().TransientBanner(messagePrefix+Lang('in progress...'))
         try:

plugins-base/XSFeatureVMCommon.py

@@ -169,7 +169,7 @@ def UpdateFieldsINITIAL(self):
         pane.ResetFields()
 
         vm = HotAccessor().guest_vm[self.vmHandle]
-        vmName = vm.name_label(None).encode('utf-8')
+        vmName = convert_anything_to_str(vm.name_label(None))
         if vmName is None:
             pane.AddTitleField(Lang("The Virtual Machine is no longer present"))
         else:
@@ -190,7 +190,7 @@ def UpdateFieldsCONFIRM(self):
         pane.ResetFields()
 
         vm = HotAccessor().vm[self.vmHandle]
-        vmName = vm.name_label(None).encode('utf-8')
+        vmName = convert_anything_to_str(vm.name_label(None))
         if vmName is None:
             pane.AddTitleField(Lang("The Virtual Machine is no longer present"))
         else:
@@ -252,7 +252,7 @@ def Commit(self):
         Layout.Inst().PopDialogue()
 
         operationName = VMUtils.OperationName(self.operation)
-        vmName = HotAccessor().guest_vm[self.vmHandle].name_label(Lang('<Unknown>')).encode('utf-8')
+        vmName = convert_anything_to_str(HotAccessor().guest_vm[self.vmHandle].name_label(Lang('<Unknown>')))
         messagePrefix = operationName + Lang(' operation on ') + vmName + ' '
         try:
             task = VMUtils.AsyncOperation(self.operation, self.vmHandle, *self.opParams)

tests/test_convert_anything_to_str.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8
+# Possible test commands:
+# PYTHONPATH=. python2 tests/test_convert_anything_to_str.py
+# PYTHONPATH=. python3 tests/test_convert_anything_to_str.py
+# python2 -m unittest discover
+# python3 -m unittest discover
+"""Test case for XSConsoleLang.convert_anything_to_str()"""
+import os
+import sys
+import unittest
+
+from XSConsoleLang import convert_anything_to_str
+
+
+class TestIPAddress(unittest.TestCase):
+    def test_convert_anything_to_str(self):
+        expected_string = "Török"
+        encoded_iso8859 = b'T\xf6r\xf6k'
+        encoded_as_utf8 = b"T\xc3\xb6r\xc3\xb6k"
+
+        assert expected_string == convert_anything_to_str(encoded_as_utf8)
+        # GitHub's Python2 is apparently a custom build that is limited:
+        if sys.version_info > (3, 0) or not os.environ.get("GITHUB_ACTION"):
+            assert expected_string == convert_anything_to_str(encoded_iso8859, "iso-8859-1")
+
+        if sys.version_info.major < 3:  # For Py2, on Py3, str == unicode
+            unicode_str = encoded_as_utf8.decode("utf-8")
+            assert str(type(unicode_str)) == "<type 'unicode'>"
+            assert expected_string == convert_anything_to_str(unicode_str)
+
+        assert expected_string == convert_anything_to_str(expected_string)
+
+        # Special case for then call location check the result for None:
+        assert convert_anything_to_str(None) == None
+
+        # Test cases for str(arg)
+        assert convert_anything_to_str(42) == "42"
+        assert convert_anything_to_str(42.2) == "42.2"
+        assert convert_anything_to_str(Exception("True")) == "True"
+        assert convert_anything_to_str(True) == "True"
+        assert convert_anything_to_str(False) == "False"
+
+
+if __name__ == "__main__":
+    unittest.main()