7166082 pkg command does not handle Japanese character encoding on Solaris 11
authorAbhinandan Ekande <abhinandan.ekande@oracle.com>
Mon, 09 Jul 2012 11:18:53 -0700
changeset 2728 69903f3b722b
parent 2727 41dcac9de303
child 2729 fbe29295150f
7166082 pkg command does not handle Japanese character encoding on Solaris 11
src/client.py
src/depot.py
src/modules/client/history.py
src/pkg/external_deps.txt
src/pkg/manifests/developer:opensolaris:pkg5.p5m
src/pkgdep.py
src/pkgrepo.py
src/publish.py
src/pull.py
src/sign.py
src/sysrepo.py
src/tests/cli/t_pkg_help.py
src/tests/ro_data/pkg.help.eucJP.expected.out
src/util/publish/pkgdiff.py
src/util/publish/pkgfmt.py
src/util/publish/pkglint.py
src/util/publish/pkgmerge.py
src/util/publish/pkgmogrify.py
--- a/src/client.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/client.py	Mon Jul 09 11:18:53 2012 -0700
@@ -6607,7 +6607,8 @@
 
 if __name__ == "__main__":
         misc.setlocale(locale.LC_ALL, "", error)
-        gettext.install("pkg", "/usr/share/locale")
+        gettext.install("pkg", "/usr/share/locale",
+            codeset=locale.getpreferredencoding())
 
         # Make all warnings be errors.
         import warnings
--- a/src/depot.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/depot.py	Mon Jul 09 11:18:53 2012 -0700
@@ -19,7 +19,7 @@
 #
 # CDDL HEADER END
 #
-# Copyright (c) 2007, 2011 Oracle and/or its affiliates.  All rights reserved.
+# Copyright (c) 2007, 2012, Oracle and/or its affiliates.  All rights reserved.
 #
 
 # pkg.depotd - package repository daemon
@@ -228,7 +228,8 @@
 if __name__ == "__main__":
 
         setlocale(locale.LC_ALL, "")
-        gettext.install("pkg", "/usr/share/locale")
+        gettext.install("pkg", "/usr/share/locale",
+            codeset=locale.getpreferredencoding())
 
         add_content = False
         exit_ready = False
--- a/src/modules/client/history.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/modules/client/history.py	Mon Jul 09 11:18:53 2012 -0700
@@ -26,8 +26,6 @@
 
 import copy
 import errno
-import gettext
-import locale
 import os
 import shutil
 import sys
@@ -104,31 +102,6 @@
     MemoryError: RESULT_FAILED_OUTOFMEMORY,
 }
 
-misc.setlocale(locale.LC_ALL, "")
-gettext.install("pkg", "/usr/share/locale")
-
-# since we store english text in our XML files, we need a way for clients
-# obtain a translated version of these messages.
-result_l10n = {
-    "Canceled": _("Canceled"),
-    "Failed": _("Failed"),
-    "Ignored": _("Ignored"),
-    "Nothing to do": _("Nothing to do"),
-    "Succeeded": _("Succeeded"),
-    "Bad Request": _("Bad Request"),
-    "Configuration": _("Configuration"),
-    "Constrained": _("Constrained"),
-    "Locked": _("Locked"),
-    "Search": _("Search"),
-    "Storage": _("Storage"),
-    "Transport": _("Transport"),
-    "Actuator": _("Actuator"),
-    "Out of Memory": _("Out of Memory"),
-    "Conflicting Actions": _("Conflicting Actions"),
-    "Unknown": _("Unknown"),
-    "None": _("None")
-}
-
 class _HistoryOperation(object):
         """A _HistoryOperation object is a representation of data about an
         operation that a pkg(5) client has performed.  This class is private
@@ -139,6 +112,8 @@
         manipulated as they are set or retrieved.
         """
 
+        result_l10n = {}    # Static variable for dictionary
+
         def __copy__(self):
                 h = _HistoryOperation()
                 for attr in ("name", "start_time", "end_time", "start_state",
@@ -235,10 +210,35 @@
         def result_text(self):
                 """Returns a tuple containing the translated text for the
                 operation result of the form (outcome, reason)."""
+
+                if not _HistoryOperation.result_l10n:
+                        # since we store english text in our XML files, we
+                        # need a way for clients obtain a translated version
+                        # of these messages.
+                        _HistoryOperation.result_l10n = {
+                            "Canceled": _("Canceled"),
+                            "Failed": _("Failed"),
+                            "Ignored": _("Ignored"),
+                            "Nothing to do": _("Nothing to do"),
+                            "Succeeded": _("Succeeded"),
+                            "Bad Request": _("Bad Request"),
+                            "Configuration": _("Configuration"),
+                            "Constrained": _("Constrained"),
+                            "Locked": _("Locked"),
+                            "Search": _("Search"),
+                            "Storage": _("Storage"),
+                            "Transport": _("Transport"),
+                            "Actuator": _("Actuator"),
+                            "Out of Memory": _("Out of Memory"),
+                            "Conflicting Actions": _("Conflicting Actions"),
+                            "Unknown": _("Unknown"),
+                            "None": _("None")
+                        }
+
                 if not self.start_time or not self.result:
                         return ("", "")
-                return (result_l10n[self.result[0]],
-                    result_l10n[self.result[1]])
+                return (_HistoryOperation.result_l10n[self.result[0]],
+                    _HistoryOperation.result_l10n[self.result[1]])
 
 
 class History(object):
--- a/src/pkg/external_deps.txt	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/pkg/external_deps.txt	Mon Jul 09 11:18:53 2012 -0700
@@ -39,6 +39,7 @@
     pkg:/system/library/dbus
     pkg:/system/library/math
     pkg:/system/linker
+    pkg:/system/locale/extra
     pkg:/system/network
     pkg:/text/gnu-grep
     pkg:/text/locale
--- a/src/pkg/manifests/developer:opensolaris:pkg5.p5m	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/pkg/manifests/developer:opensolaris:pkg5.p5m	Mon Jul 09 11:18:53 2012 -0700
@@ -48,6 +48,7 @@
 depend type=require fmri=pkg:/runtime/python-27
 depend type=require fmri=pkg:/system/header
 depend type=require fmri=pkg:/system/linker
+depend type=require fmri=pkg:/system/locale/extra
 depend type=require fmri=pkg:/text/locale
 depend type=require fmri=pkg:/text/tidy
 depend type=require fmri=pkg:/web/server/apache-22
--- a/src/pkgdep.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/pkgdep.py	Mon Jul 09 11:18:53 2012 -0700
@@ -535,7 +535,8 @@
 
 def main_func():
         misc.setlocale(locale.LC_ALL, "", error)
-        gettext.install("pkg", "/usr/share/locale")
+        gettext.install("pkg", "/usr/share/locale",
+            codeset=locale.getpreferredencoding())
 
         try:
                 opts, pargs = getopt.getopt(sys.argv[1:], "R:?",
--- a/src/pkgrepo.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/pkgrepo.py	Mon Jul 09 11:18:53 2012 -0700
@@ -1290,7 +1290,8 @@
 
 if __name__ == "__main__":
         misc.setlocale(locale.LC_ALL, "", error)
-        gettext.install("pkg", "/usr/share/locale")
+        gettext.install("pkg", "/usr/share/locale",
+            codeset=locale.getpreferredencoding())
 
         # Make all warnings be errors.
         warnings.simplefilter('error')
--- a/src/publish.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/publish.py	Mon Jul 09 11:18:53 2012 -0700
@@ -27,6 +27,7 @@
 import fnmatch
 import getopt
 import gettext
+import locale
 import os
 import sys
 import traceback
@@ -695,7 +696,8 @@
         return xport, targ_pub
 
 def main_func():
-        gettext.install("pkg", "/usr/share/locale")
+        gettext.install("pkg", "/usr/share/locale",
+            codeset=locale.getpreferredencoding())
 
         repo_uri = os.getenv("PKG_REPO", None)
 
--- a/src/pull.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/pull.py	Mon Jul 09 11:18:53 2012 -0700
@@ -28,6 +28,7 @@
 import errno
 import getopt
 import gettext
+import locale
 import os
 import shutil
 import sys
@@ -359,7 +360,8 @@
 
         temp_root = misc.config_temp_root()
 
-        gettext.install("pkg", "/usr/share/locale")
+        gettext.install("pkg", "/usr/share/locale",
+            codeset=locale.getpreferredencoding())
 
         global_settings.client_name = "pkgrecv"
         target = os.environ.get("PKG_DEST", None)
--- a/src/sign.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/sign.py	Mon Jul 09 11:18:53 2012 -0700
@@ -21,7 +21,7 @@
 #
 
 #
-# Copyright (c) 2010, 2011, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved.
 #
 
 import getopt
@@ -114,7 +114,8 @@
 
 def main_func():
         misc.setlocale(locale.LC_ALL, "", error)
-        gettext.install("pkg", "/usr/share/locale")
+        gettext.install("pkg", "/usr/share/locale",
+            codeset=locale.getpreferredencoding())
         global_settings.client_name = "pkgsign"
 
         try:
--- a/src/sysrepo.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/sysrepo.py	Mon Jul 09 11:18:53 2012 -0700
@@ -919,7 +919,8 @@
 
 if __name__ == "__main__":
         misc.setlocale(locale.LC_ALL, "", error)
-        gettext.install("pkg", "/usr/share/locale")
+        gettext.install("pkg", "/usr/share/locale",
+            codeset=locale.getpreferredencoding())
 
         # Make all warnings be errors.
         warnings.simplefilter('error')
--- a/src/tests/cli/t_pkg_help.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/tests/cli/t_pkg_help.py	Mon Jul 09 11:18:53 2012 -0700
@@ -20,16 +20,21 @@
 # CDDL HEADER END
 #
 
-# Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2009, 2012, Oracle and/or its affiliates. All rights reserved.
 
 import testutils
 if __name__ == "__main__":
         testutils.setup_environment("../../../proto")
 import pkg5unittest
 
+import codecs
+import os
+import re
 import unittest
 
 class TestPkgHelp(pkg5unittest.CliTestCase):
+        # Tests in this suite use the read only data directory.
+        need_ro_data = True
 
         def test_help(self):
                 """Verify that usage message works regardless of how it is
@@ -83,5 +88,46 @@
                             "pkg [options] command [cmd_options] [operands]",
                             "PKG_IMAGE"])
 
+        def test_help_character_encoding(self):
+                """Verify help command output for ja_JP.eucJP locale.
+                Match against the expected output"""
+
+                # This is a test case for CR 7166082.
+                # It compares the output of "pkg --help" command against
+                # the expected output for ja_JP.eucJP locale.
+                # If first 4 lines of "pkg --help" command output modified
+                # in the future then this test case will also need to be
+                # modified.
+
+                ret, out = self.cmdline_run("/usr/bin/locale -a", out=True)
+                line = " ".join(out.split())
+                m = re.search(r"ja_JP.eucJP", line)
+                self.assert_(m, "You must have ja_JP.eucJP"
+                    " locale installed for this test to succeed.")
+
+                eucJP_encode_file = os.path.join(self.ro_data_root,
+                    "pkg.help.eucJP.expected.out")
+                f = codecs.open(eucJP_encode_file, encoding="eucJP")
+
+                locale_env = { "LC_ALL": "ja_JP.eucJP" }
+                ret, out, err = self.pkg("--help", env_arg=locale_env,
+                    out=True, stderr=True)
+                cmd_out = unicode(err, encoding="eucJP")
+                # Take only 4 lines from "pkg --help" command output.
+                u_out = cmd_out.splitlines()[:4]
+
+                n = 0
+                # The expected output file contain 4 lines and command output
+                # is also 4 lines.
+                while n < 4:
+                        cmd_line = u_out[n]
+                        # Remove \n from readline()
+                        file_line = f.readline()[:-1]
+
+                        self.assertEqual(cmd_line, file_line)
+                        n = n + 1
+
+                f.close()
+
 if __name__ == "__main__":
         unittest.main()
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/tests/ro_data/pkg.help.eucJP.expected.out	Mon Jul 09 11:18:53 2012 -0700
@@ -0,0 +1,4 @@
+������ˡ:
+        pkg [options] command [cmd_options] [operands]
+
+����Ū�ʥ��֥��ޥ��:
--- a/src/util/publish/pkgdiff.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/util/publish/pkgdiff.py	Mon Jul 09 11:18:53 2012 -0700
@@ -26,6 +26,7 @@
 
 import getopt
 import gettext
+import locale
 import sys
 import traceback
 
@@ -59,7 +60,8 @@
                 sys.exit(exitcode)
 
 def main_func():
-        gettext.install("pkg", "/usr/share/locale")
+        gettext.install("pkg", "/usr/share/locale",
+            codeset=locale.getpreferredencoding())
 
         ignoreattrs = []
         onlyattrs = []
--- a/src/util/publish/pkgfmt.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/util/publish/pkgfmt.py	Mon Jul 09 11:18:53 2012 -0700
@@ -20,7 +20,7 @@
 # CDDL HEADER END
 
 #
-# Copyright (c) 2009, 2011, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2009, 2012, Oracle and/or its affiliates. All rights reserved.
 #
 
 # Prefixes should be ordered alphabetically with most specific first.
@@ -56,6 +56,7 @@
         import errno
         import getopt
         import gettext
+        import locale
         import operator
         import os
         import re
@@ -489,7 +490,8 @@
         print >> fileobj, output
 
 def main_func():
-        gettext.install("pkg", "/usr/share/locale")
+        gettext.install("pkg", "/usr/share/locale",
+            codeset=locale.getpreferredencoding())
         global opt_unwrap
         global opt_check
         global opt_diffs
--- a/src/util/publish/pkglint.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/util/publish/pkglint.py	Mon Jul 09 11:18:53 2012 -0700
@@ -21,18 +21,17 @@
 #
     
 #
-# Copyright (c) 2010, 2011 Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved.
 #
 
 import codecs
 import logging
 import sys
 import gettext
+import locale
 import traceback
 from optparse import OptionParser
 
-gettext.install("pkg", "/usr/share/locale")
-
 from pkg.client.api_errors import InvalidPackageErrors
 from pkg import VERSION
 from pkg.misc import PipeError
@@ -57,6 +56,9 @@
 def main_func():
         """Start pkglint."""
 
+        gettext.install("pkg", "/usr/share/locale",
+            codeset=locale.getpreferredencoding())
+
         global logger
         
         usage = \
--- a/src/util/publish/pkgmerge.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/util/publish/pkgmerge.py	Mon Jul 09 11:18:53 2012 -0700
@@ -859,7 +859,8 @@
 
 if __name__ == "__main__":
         misc.setlocale(locale.LC_ALL, "", error)
-        gettext.install("pkg", "/usr/share/locale")
+        gettext.install("pkg", "/usr/share/locale",
+            codeset=locale.getpreferredencoding())
 
         # Make all warnings be errors.
         import warnings
--- a/src/util/publish/pkgmogrify.py	Mon Jul 09 15:47:03 2012 -0700
+++ b/src/util/publish/pkgmogrify.py	Mon Jul 09 11:18:53 2012 -0700
@@ -20,11 +20,12 @@
 # CDDL HEADER END
 
 #
-# Copyright (c) 2009, 2011, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2009, 2012, Oracle and/or its affiliates. All rights reserved.
 #
 
 import getopt
 import gettext
+import locale
 import os
 import re
 import shlex
@@ -661,7 +662,8 @@
                 sys.exit(exitcode)
 
 def main_func():
-        gettext.install("pkg", "/usr/share/locale")
+        gettext.install("pkg", "/usr/share/locale",
+            codeset=locale.getpreferredencoding())
 
         outfilename = None
         printfilename = None