pwclient: Fix encoding problems

All data returned by the xmlrpc object is unicode decoded with 'utf-8' (on
python 3, unicode == str). Add from __future__ import unicode_literals
to make sure that everything is unicode and avoid surprises.

On python 2, printing unicode to stdout causes it to be encoded to str
(byte string) with the 'ascii' codec:

  >>> print some_unicode_string
  ...
  UnicodeEncodeError: 'ascii' codec can't encode character u'\u0142'
  in position 468: ordinal not in range(128)

Work around ths by avoiding any explicit call to unicode() and by
replacing sys.stdout and sys.stderr by unicode-aware file objects (as
returned by io.open()).

Guess the encoding of stdout and stderr by looking at (in that order):
sys.stdout.encoding, locale.getpreferredencoding(), the PYTHONIOENCODING
environment variable. If no encoding is defined, assume 'utf-8' as
output encoding.

Conflicts:
    patchwork/bin/pwclient

Tested-by: Thomas Monjalon <thomas.monjalon@6wind.com>
Reviewed-by: Stephen Finucane <stephen@that.guru>
(cherry picked from commit 046419a3bf8f012fcb019c3ec8d78653da685908)

diff --git a/patchwork/bin/pwclient b/patchwork/bin/pwclient
index 3d5be691994e30ca95b8a0e21deff2d5efca995d..808c2b94c62fd86cebfc46ab584917309947f1e8 100755 (executable)
--- a/patchwork/bin/pwclient
+++ b/patchwork/bin/pwclient
@@ -21,6 +21,7 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os
 import sys
@@ -40,13 +41,17 @@ except ImportError:
     import configparser as ConfigParser
 import shutil
 import re
-
-# Add a shim for Python 2's unicode() helper.
-try:
-    unicode
-except NameError:
-    # Python 3 does everything by unicode now.
-    unicode = str
+import io
+import locale
+
+if sys.version_info.major == 2:
+    # hack to make writing unicode to standard output/error work on Python 2
+    OUT_ENCODING = sys.stdout.encoding or locale.getpreferredencoding() or \
+            os.getenv('PYTHONIOENCODING', 'utf-8')
+    sys.stdout = io.open(sys.stdout.fileno(), mode='w',
+                         encoding=OUT_ENCODING, errors='replace')
+    sys.stderr = io.open(sys.stderr.fileno(), mode='w',
+                         encoding=OUT_ENCODING, errors='replace')
 
 # Default Patchwork remote XML-RPC server URL
 # This script will check the PW_XMLRPC_URL environment variable
@@ -214,8 +219,7 @@ def action_list(rpc, filter, submitter_str, delegate_str, format_str=None):
             for id in ids:
                 person = rpc.person_get(id)
                 print('Patches submitted by %s <%s>:' %
-                      (unicode(person['name']).encode('utf-8'),
-                       unicode(person['email']).encode('utf-8')))
+                      (person['name'], person['email']))
                 f = filter
                 f.add("submitter_id", id)
                 patches = rpc.patch_list(f.d)
@@ -269,7 +273,7 @@ def action_check_info(rpc, check_id):
     print(s)
     print('-' * len(s))
     for key, value in sorted(check.items()):
-        print("- %- 14s: %s" % (key, unicode(value)))
+        print("- %- 14s: %s" % (key, value))
 
 
 def action_check_create(rpc, patch_id, context, state, url, description):
@@ -293,7 +297,7 @@ def action_info(rpc, patch_id):
     print(s)
     print('-' * len(s))
     for key, value in sorted(patch.items()):
-        print("- %- 14s: %s" % (key, unicode(value)))
+        print("- %- 14s: %s" % (key, value))
 
 
 def action_get(rpc, patch_id):
@@ -311,7 +315,7 @@ def action_get(rpc, patch_id):
         i += 1
 
     try:
-        f = open(fname, "w")
+        f = io.open(fname, "w", encoding="utf-8")
     except:
         sys.stderr.write("Unable to open %s for writing\n" % fname)
         sys.exit(1)
@@ -343,7 +347,7 @@ def action_apply(rpc, patch_id, apply_cmd=None):
     s = rpc.patch_get_mbox(patch_id)
     if len(s) > 0:
         proc = subprocess.Popen(apply_cmd, stdin=subprocess.PIPE)
-        proc.communicate(unicode(s).encode('utf-8'))
+        proc.communicate(s.encode('utf-8'))
         return proc.returncode
     else:
         sys.stderr.write("Error: No patch content found\n")
@@ -757,7 +761,7 @@ def main():
             for patch_id in non_empty(h, patch_ids):
                 s = rpc.patch_get_mbox(patch_id)
                 if len(s) > 0:
-                    i.append(unicode(s))
+                    i.append(s)
             if len(i) > 0:
                 pager.communicate(input="\n".join(i).encode("utf-8"))
             pager.stdin.close()
@@ -765,7 +769,7 @@ def main():
             for patch_id in non_empty(h, patch_ids):
                 s = rpc.patch_get_mbox(patch_id)
                 if len(s) > 0:
-                    print(unicode(s))
+                    print(s)
 
     elif action == 'info':
         for patch_id in non_empty(h, patch_ids):