PSARC/2014/113 xattr: filesystem extended attribute support for Python
authorDanek Duvall <danek.duvall@oracle.com>
Tue, 22 Apr 2014 12:43:58 -0700
changeset 1847 b43426a2f6ba
parent 1846 df40919e04fa
child 1848 289daf750b26
PSARC/2014/113 xattr: filesystem extended attribute support for Python 18468627 integrate xattr
components/python/xattr/Makefile
components/python/xattr/patches/build.patch
components/python/xattr/patches/solaris.patch
components/python/xattr/patches/test.patch
components/python/xattr/resolve.deps
components/python/xattr/xattr-PYVER.p5m
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/components/python/xattr/Makefile	Tue Apr 22 12:43:58 2014 -0700
@@ -0,0 +1,85 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+#
+
+include ../../../make-rules/shared-macros.mk
+
+COMPONENT_NAME=		xattr
+COMPONENT_VERSION=	0.7.4
+COMPONENT_SRC=		$(COMPONENT_NAME)-$(COMPONENT_VERSION)
+COMPONENT_ARCHIVE=	$(COMPONENT_SRC).tar.gz
+COMPONENT_ARCHIVE_HASH=	\
+    sha256:16c612ac01b9b5a31967e231c7f7d32b7263f151d5e3c617c69378170054805f
+COMPONENT_ARCHIVE_URL=	$(call pypi_url)
+COMPONENT_PROJECT_URL=	http://github.com/xattr/xattr
+COMPONENT_BUGDB=	python-mod/xattr
+
+include $(WS_TOP)/make-rules/prep.mk
+include $(WS_TOP)/make-rules/setup.py.mk
+include $(WS_TOP)/make-rules/ips.mk
+
+ASLR_MODE = $(ASLR_NOT_APPLICABLE)
+
+# Look over into the pycparser and cffi proto areas until they're available in
+# the build environment.  Make sure that cffi is built.
+PYCPARSER =		$(WS_COMPONENTS)/python/pycparser/build/prototype/$(MACH)/$(PYTHON_LIB)
+CFFI =			$(WS_COMPONENTS)/python/cffi/build/prototype/$(MACH)/$(PYTHON_LIB)
+
+$(CFFI):
+	(cd ../cffi; $(MAKE) install)
+
+PYTHONPATH_DEP =	$(PYCPARSER):$(CFFI)
+COMPONENT_BUILD_ENV +=	PYTHONPATH=$(PYTHONPATH_DEP)
+COMPONENT_INSTALL_ENV +=	PYTHONPATH=$(PYTHONPATH_DEP):$(PROTO_DIR)/$(PYTHON_LIB)
+COMPONENT_TEST_ENV +=	PYTHONPATH=$(PYTHONPATH_DEP):$(PROTO_DIR)/$(PYTHON_LIB)
+# The test suite wants to use some unicode filenames, which don't work in the C
+# locale.
+COMPONENT_TEST_ENV +=	LC_CTYPE=en_US.UTF-8
+COMPONENT_TEST_DIR =	$(@D)/tests
+COMPONENT_TEST_CMD =	$(PYTHON) setup.py
+COMPONENT_TEST_ARGS =	test
+
+COMPONENT_POST_INSTALL_ACTION += \
+	(cd $(PROTO_DIR)/usr/bin; $(MV) xattr xattr-$(PYTHON_VERSION))
+
+# Tests run under setuptools work very hard to run in the source directory
+# itself, and will thus blow up with an VerificationError (wrong ELF class)
+# when running 64-bit tests after 32-bit or vice-versa.  So copy the source off
+# to a target-specific directory and run the tests.  Note that this will also
+# rebuild the package, since it doesn't want to use what it should already be
+# finding in PYTHONPATH.
+COMPONENT_PRE_TEST_ACTION = $(CP) -r $(SOURCE_DIR) $(@D)/tests
+
+$(BUILD_32_and_64): $(CFFI)
+
+# common targets
+build:		$(BUILD_32_and_64)
+
+install:	$(INSTALL_32_and_64)
+
+test:		$(TEST_32_and_64)
+
+BUILD_PKG_DEPENDENCIES =	$(BUILD_TOOLS)
+
+include $(WS_TOP)/make-rules/depend.mk
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/components/python/xattr/patches/build.patch	Tue Apr 22 12:43:58 2014 -0700
@@ -0,0 +1,21 @@
+cffi leaves __pycache__ around in the *source* directory, so when we build
+64-bit after 32-bit (or vice versa), the modules already exist, but are of the
+wrong ELF class.  Ideally, the entire build would happen outside the source
+directory, but for the time being, this patch at least makes sure distutils'
+build_base directory (plus an extraneous directory structure) is used for the
+object files and shared objects.
+
+There's probably a better way, but it will likely involve deep distutils magic
+and maybe some patches to cffi.
+
+--- xattr-0.7.4/setup.py	Mon Mar  3 10:13:29 2014
++++ xattr-0.7.4/setup.py	Mon Apr 14 14:09:58 2014
+@@ -12,6 +12,8 @@
+     setup_requires hasn't run yet.
+     """
+     def finalize_options(self):
++        import cffi.verifier
++        cffi.verifier.set_tmpdir(self.build_base)
+         from xattr.lib import ffi
+         self.distribution.ext_modules = [ffi.verifier.get_extension()]
+         build.finalize_options(self)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/components/python/xattr/patches/solaris.patch	Tue Apr 22 12:43:58 2014 -0700
@@ -0,0 +1,78 @@
+Patch needed for xattr to build properly on Solaris, with Studio compilers.
+
+Merged as of 0.7.6.
+
+diff --git a/xattr/lib.py b/xattr/lib.py
+index 0466d30..390c333 100644
+--- a/xattr/lib.py
++++ b/xattr/lib.py
+@@ -29,12 +29,13 @@ lib = ffi.verify("""
+ #include "Python.h"
+ #ifdef __FreeBSD__
+ #include <sys/extattr.h>
+-#elif defined(__SUN__) || defined(__sun__) || defined(sun)
++#elif defined(__SUN__) || defined(__sun__) || defined(__sun)
+ #include <sys/types.h>
+ #include <sys/stat.h>
+ #include <fcntl.h>
+ #include <unistd.h>
+ #include <dirent.h>
++#include <alloca.h>
+ #else
+ #include <sys/xattr.h>
+ #endif
+@@ -59,7 +60,7 @@ static void convert_bsd_list(char *namebuf, size_t size)
+     while(offset < size) {
+         int length = (int) namebuf[offset];
+         memmove(namebuf+offset, namebuf+offset+1, length);
+-        namebuf[offset+length] = '\0';
++        namebuf[offset+length] = '\\0';
+         offset += length+1;
+     }
+ }
+@@ -260,7 +261,7 @@ static ssize_t xattr_flistxattr(int fd, char *namebuf, size_t size, int options)
+     return rv;
+ }
+ 
+-#elif defined(__SUN__) || defined(__sun__) || defined(sun)
++#elif defined(__SUN__) || defined(__sun__) || defined(__sun)
+ 
+ /* Solaris 9 and later compatibility API */
+ #define XATTR_XATTR_NOFOLLOW 0x0001
+@@ -268,6 +269,9 @@ static ssize_t xattr_flistxattr(int fd, char *namebuf, size_t size, int options)
+ #define XATTR_XATTR_REPLACE 0x0004
+ #define XATTR_XATTR_NOSECURITY 0x0008
+ 
++#define XATTR_CREATE 0x1
++#define XATTR_REPLACE 0x2
++
+ #ifndef u_int32_t
+ #define u_int32_t uint32_t
+ #endif
+@@ -429,7 +433,7 @@ static ssize_t xattr_xflistxattr(int xfd, char *namebuf, size_t size, int option
+             snprintf((char *)(namebuf + nsize), esize + 1,
+                     entry->d_name);
+         }
+-        nsize += esize + 1; /* +1 for \0 */
++        nsize += esize + 1; /* +1 for \\0 */
+     }
+     closedir(dirp);
+     return nsize;
+@@ -438,7 +442,7 @@ static ssize_t xattr_flistxattr(int fd, char *namebuf, size_t size, int options)
+ {
+     int xfd;
+ 
+-    xfd = openat(fd, ".", O_RDONLY);
++    xfd = openat(fd, ".", O_RDONLY | O_XATTR);
+     return xattr_xflistxattr(xfd, namebuf, size, options);
+ }
+ 
+@@ -716,7 +720,7 @@ def _flistxattr(fd, options=0):
+     flistxattr(fd, options=0) -> str
+     """
+     res = lib.xattr_flistxattr(fd, ffi.NULL, 0, options)
+-    if res == 1:
++    if res == -1:
+         raise error()
+     buf = ffi.new("char[]", res)
+     res = lib.xattr_flistxattr(fd, buf, res, options)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/components/python/xattr/patches/test.patch	Tue Apr 22 12:43:58 2014 -0700
@@ -0,0 +1,48 @@
+Every file on Solaris 11 and on has two extended attributes, so if we find
+them, we shouldn't fail.  Also, we don't support extended attributes on
+symlinks, so just quietly pass that test.
+
+Merged as of 0.7.6.
+
+diff --git a/xattr/tests/test_xattr.py b/xattr/tests/test_xattr.py
+--- a/xattr/tests/test_xattr.py	Mon Mar  3 10:12:24 2014
++++ b/xattr/tests/test_xattr.py	Wed Mar 26 16:29:24 2014
+@@ -1,4 +1,5 @@
+ import os
++import sys
+ from unittest import TestCase
+ from tempfile import mkdtemp, NamedTemporaryFile
+ 
+@@ -8,10 +9,19 @@
+ class BaseTestXattr(object):
+     def test_attr(self):
+         x = xattr.xattr(self.tempfile)
+-        self.assertEqual(x.keys(), [])
+-        self.assertEqual(x.list(), [])
+-        self.assertEqual(dict(x), {})
+ 
++        # Solaris 11 and forward contain system attributes (file flags) in
++        # extended attributes present on all files, so cons them up into a
++        # comparison dict.
++        d = {}
++        if sys.platform == 'sunos5' and 'SUNWattr_ro' in x:
++            d['SUNWattr_ro'] = x['SUNWattr_ro']
++            d['SUNWattr_rw'] = x['SUNWattr_rw']
++
++        self.assertEqual(x.keys(), d.keys())
++        self.assertEqual(x.list(), d.keys())
++        self.assertEqual(dict(x), d)
++
+         x['user.sopal'] = b'foo'
+         x['user.sop.foo'] = b'bar'
+         x[u'user.\N{SNOWMAN}'] = b'not a snowman'
+@@ -38,6 +48,9 @@
+         self.assertTrue('user.sop.foo' not in x)
+ 
+     def test_symlink_attrs(self):
++        # Solaris doesn't support extended attributes on symlinks
++        if sys.platform == 'sunos5':
++            return
+         symlinkPath = self.tempfilename + '.link'
+         os.symlink(self.tempfilename, symlinkPath)
+         try:
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/components/python/xattr/resolve.deps	Tue Apr 22 12:43:58 2014 -0700
@@ -0,0 +1,4 @@
+library/python/setuptools-26
+library/python/setuptools-27
+runtime/python-26
+runtime/python-27
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/components/python/xattr/xattr-PYVER.p5m	Tue Apr 22 12:43:58 2014 -0700
@@ -0,0 +1,67 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+#
+
+set name=pkg.fmri \
+    value=pkg:/library/python/xattr-$(PYV)@$(IPS_COMPONENT_VERSION),$(BUILD_VERSION)
+set name=pkg.summary value="Python wrapper for extended filesystem attributes"
+set name=com.oracle.info.description value="the xattr Python module"
+set name=com.oracle.info.tpno value=16912
+set name=info.classification \
+    value=org.opensolaris.category.2008:Development/Python
+set name=info.source-url value=$(COMPONENT_ARCHIVE_URL)
+set name=info.upstream value="Bob Ippolito <[email protected]>"
+set name=info.upstream-url value=$(COMPONENT_PROJECT_URL)
+set name=org.opensolaris.arc-caseid value=PSARC/2014/113
+set name=org.opensolaris.consolidation value=$(CONSOLIDATION)
+#
+link path=usr/bin/xattr target=xattr-$(PYVER) mediator=python \
+    mediator-version=$(PYVER)
+file path=usr/bin/xattr-$(PYVER)
+file path=usr/lib/python$(PYVER)/vendor-packages/xattr-$(COMPONENT_VERSION)-py$(PYVER).egg-info/PKG-INFO
+file path=usr/lib/python$(PYVER)/vendor-packages/xattr-$(COMPONENT_VERSION)-py$(PYVER).egg-info/SOURCES.txt
+file path=usr/lib/python$(PYVER)/vendor-packages/xattr-$(COMPONENT_VERSION)-py$(PYVER).egg-info/dependency_links.txt
+file path=usr/lib/python$(PYVER)/vendor-packages/xattr-$(COMPONENT_VERSION)-py$(PYVER).egg-info/entry_points.txt
+file path=usr/lib/python$(PYVER)/vendor-packages/xattr-$(COMPONENT_VERSION)-py$(PYVER).egg-info/not-zip-safe
+file path=usr/lib/python$(PYVER)/vendor-packages/xattr-$(COMPONENT_VERSION)-py$(PYVER).egg-info/requires.txt
+file path=usr/lib/python$(PYVER)/vendor-packages/xattr-$(COMPONENT_VERSION)-py$(PYVER).egg-info/top_level.txt
+$(PYTHON_2.6_ONLY)file path=usr/lib/python$(PYVER)/vendor-packages/xattr/64/_cffi__x65092e53xe621c3cb.so
+$(PYTHON_2.7_ONLY)file path=usr/lib/python$(PYVER)/vendor-packages/xattr/64/_cffi__x93e1032axe621c3cb.so
+file path=usr/lib/python$(PYVER)/vendor-packages/xattr/__init__.py
+$(PYTHON_2.6_ONLY)file path=usr/lib/python$(PYVER)/vendor-packages/xattr/_cffi__x65092e53xe621c3cb.so
+$(PYTHON_2.7_ONLY)file path=usr/lib/python$(PYVER)/vendor-packages/xattr/_cffi__x93e1032axe621c3cb.so
+file path=usr/lib/python$(PYVER)/vendor-packages/xattr/lib.py
+file path=usr/lib/python$(PYVER)/vendor-packages/xattr/tool.py
+#
+license LICENSE.txt license=MIT
+
+# Dependencies on cffi must be incorporated.
+depend type=incorporate fmri=library/python/cffi-$(PYV)@0.8.2
+
+# force a dependency on cffi; pkgdepend work is needed to flush this out.
+depend type=require fmri=library/python/cffi-$(PYV)
+
+# force a dependency on the xattr package
+depend type=require \
+    fmri=library/python/xattr@$(IPS_COMPONENT_VERSION),$(BUILD_VERSION)