tools/userland-mangler
author Lijo George<lijo.x.george@oracle.com>
Thu, 26 Nov 2015 02:23:48 -0800
changeset 5129 5431772f7235
parent 4973 be26c0c14fa7
child 5175 c63cda9960d4
permissions -rwxr-xr-x
PSARC/2014/162 ksh93 update to 2012-08-01 17533968 ksh93 uprev to latest community version 17817727 ksh93: Right shift arithmetic substitution error for shifts of 64 bits or more 17699248 ksh93 double associative array handling bugs 17777549 "kill %%" with no background jobs , coredumps 18119738 ksh93 crashes in sfio area 18229654 ksh93 read not reentrant in alarm context dumps core 16169978 ksh93 memory corruption with redirection 18302723 ksh93 segv in sh_setmatch 16507675 external command in double-nested here-document hangs ksh93 18920300 remove pkglint Warnings in ksh93 build 18355790 /usr/bin/sh and /usr/sbin/sh should point to /usr/bin/ksh93 19907453 Session drop can cause ksh93 to become a fork bomb 18426052 SPARC /usr/bin/ksh is not an XPG6 executable 20808157 attpackagemake.mk test target needs the same environment as the build 20948390 ksh93 should have some master test results to compare against 20948350 attpackagemake.mk tested-and-compared target has mis-matched parentheses

#!/usr/bin/python2.7
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
# Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved.
#
#
# userland-mangler - a file mangling utility
#
#  A simple program to mangle files to conform to Solaris WOS or Consoldation
#  requirements.
#

import os
import sys
import re

import pkg.fmri
import pkg.manifest
import pkg.actions
import pkg.elf as elf

attribute_oracle_table_header = """
.\\\" Oracle has added the ARC stability level to this manual page"""

attribute_table_header = """
.SH ATTRIBUTES
See
.BR attributes (5)
for descriptions of the following attributes:
.sp
.TS
box;
cbp-1 | cbp-1
l | l .
ATTRIBUTE TYPE	ATTRIBUTE VALUE """

attribute_table_availability = """
=
Availability	%s"""

attribute_table_stability = """
=
Stability	%s"""

attribute_table_footer = """
.TE 
.PP
"""
def attributes_section_text(availability, stability, modified_date):
	result = ''

	# is there anything to do?
	if availability is not None or stability is not None:
		result = attribute_oracle_table_header
		if modified_date is not None:
			result += ("\n.\\\" on %s" % modified_date)
		result += attribute_table_header

		if availability is not None:
			result += (attribute_table_availability % availability)
		if stability is not None:
			result += (attribute_table_stability % stability.capitalize())
		result += attribute_table_footer

	return result

notes_oracle_comment = """
.\\\" Oracle has added source availability information to this manual page"""

notes_header = """
.SH NOTES
"""

notes_community = """
Further information about this software can be found on the open source community website at %s.
"""
notes_source = """
This software was built from source available at https://java.net/projects/solaris-userland.  The original community source was downloaded from  %s
"""

def notes_section_text(header_seen, community, source, modified_date):
	result = ''

	# is there anything to do?
	if community is not None or source is not None:
		if header_seen == False:
			result += notes_header
		result += notes_oracle_comment
		if modified_date is not None:
			result += ("\n.\\\" on %s" % modified_date)
		if source is not None:
			result += (notes_source % source)
		if community is not None:
			result += (notes_community % community)

	return result

so_re = re.compile('^\.so.+$', re.MULTILINE)
section_re = re.compile('\.SH "?([^"]+).*$', re.IGNORECASE)
TH_re = re.compile('\.TH\s+(?:"[^"]+"|\S+)\s+(\S+)', re.IGNORECASE)
#
# mangler.man.stability = (mangler.man.stability)
# mangler.man.modified_date = (mangler.man.modified-date)
# mangler.man.availability = (pkg.fmri)
# mangler.man.source-url = (pkg.source-url)
# mangler.man.upstream-url = (pkg.upstream-url)
# mangler.man.rewrite-section = ('true'/'false')   default 'true'
#
def mangle_manpage(manifest, action, text):
	# manpages must have a taxonomy defined
	stability = action.attrs.pop('mangler.man.stability', None)
	if stability is None:
		sys.stderr.write("ERROR: manpage action missing mangler.man.stability: %s" % action)
		sys.exit(1)

	# manpages may have a 'modified date'
	modified_date = action.attrs.pop('mangler.man.modified-date', None)


        # Rewrite the section in the .TH line to match the section in which
        # we're delivering it.
        rewrite_sect = action.attrs.pop('mangler.man.rewrite-section', 'true')

	attributes_written = False
	notes_seen = False

	if 'pkg.fmri' in manifest.attributes:
		fmri = pkg.fmri.PkgFmri(manifest.attributes['pkg.fmri'])
		availability = fmri.pkg_name

	community = None
	if 'info.upstream-url' in manifest.attributes:
		community = manifest.attributes['info.upstream-url']

	source = None
	if 'info.source-url' in manifest.attributes:
		source = manifest.attributes['info.source-url']
	elif 'info.repository-url' in manifest.attributes:
		source = manifest.attributes['info.repository-url']

	# skip reference only pages
	if so_re.match(text) is not None:
		return text

	# tell man that we want tables (and eqn)
	result = "'\\\" te\n"

	# write the orginal data
	for line in text.split('\n'):
		match = section_re.match(line)
		if match is not None:
			section = match.group(1)
			if section in ['SEE ALSO', 'NOTES']:
				if attributes_written == False:
					result += attributes_section_text(
								 availability,
								 stability,
								 modified_date)
					attributes_written = True
				if section == 'NOTES':
					notes_seen = True

                match = TH_re.match(line)
                if match and rewrite_sect.lower() == "true":
                        # Use the section defined by the filename, rather than
                        # the directory in which it sits.
                        sect = os.path.splitext(action.attrs["path"])[1][1:]
                        line = line[:match.span(1)[0]] + sect + \
                            line[match.span(1)[1]:]

		result += ("%s\n" % line)

	if attributes_written == False:
		result += attributes_section_text(availability, stability,
		    modified_date)

	result += notes_section_text(notes_seen, community, source,
	    modified_date)

	return result


#
# mangler.elf.strip = (true|false)
#
def mangle_elf(manifest, action, src, dest):
	pass

#
# mangler.script.file-magic =
#
def mangle_script(manifest, action, text):
	return text

#
# mangler.strip_cddl = false
#
def mangle_cddl(manifest, action, text):
	strip_cddl = action.attrs.pop('mangler.strip_cddl', 'true')
	if strip_cddl is 'false':
		return text
	cddl_re = re.compile('^[^\n]*CDDL HEADER START.+CDDL HEADER END[^\n]*\n',
			     re.MULTILINE|re.DOTALL)
	return cddl_re.sub('', text)

def mangle_path(manifest, action, src, dest):
	if elf.is_elf_object(src):
		mangle_elf(manifest, action, src, dest)
	else:
		# a 'text' document (script, man page, config file, ...
		ifp = open(src, 'r')
		text = ifp.read()
		ifp.close()

		# remove the CDDL from files
		result = mangle_cddl(manifest, action, text)

		if 'facet.doc.man' in action.attrs:
		 	result = mangle_manpage(manifest, action, result)
		elif 'mode' in action.attrs and int(action.attrs['mode'], 8) & 0111 != 0:
			result = mangle_script(manifest, action, result)

		if text != result:
			destdir = os.path.dirname(dest)
			if not os.path.exists(destdir):
				os.makedirs(destdir)
			with open(dest, 'w') as ofp:
			    ofp.write(result)

#
# mangler.bypass = (true|false)
#
def mangle_paths(manifest, search_paths, destination):
	for action in manifest.gen_actions_by_type("file"):
		bypass = action.attrs.pop('mangler.bypass', 'false').lower()
		if bypass == 'true':
			continue

		path = None
		if 'path' in action.attrs:
			path = action.attrs['path']
		if action.hash and action.hash != 'NOHASH':
			path = action.hash
		if not path:
			continue

		if not os.path.exists(destination):
			os.makedirs(destination)

		dest = os.path.join(destination, path)
		for directory in search_paths:
			if directory != destination:
				src = os.path.join(directory, path)
				if os.path.isfile(src):
					mangle_path(manifest, action, src, dest)
					break

def mangle_manifest(manifest):
	# Check for file content and remove tpno data and license actions if
	# there is no content in the package that can be licensed.
	manifest_has_file_content = False

	for action in manifest.gen_actions_by_type("file"):
		manifest_has_file_content = True
		break

	if not manifest_has_file_content:
		# search for and remove 'set name=com.oracle.info.tpno ...'
		for action in manifest.gen_actions_by_type("set"):
			if (action.attrs["name"] == "com.oracle.info.tpno"):
				manifest.actions.remove(action)
		for action in manifest.gen_actions_by_type("license"):
			manifest.actions.remove(action)

	# Check for pkg.obsolete and if found, remove any depend actions.
	manifest_is_obsolete = False

	for action in manifest.gen_actions_by_type("set"):
		if (action.attrs["name"] == "pkg.obsolete" and
		    action.attrs["value"] == "true"):
			manifest_is_obsolete = True

	if manifest_is_obsolete:
		for action in manifest.gen_actions_by_type("depend"):
			manifest.actions.remove(action)

def load_manifest(manifest_file):
	manifest = pkg.manifest.Manifest()
	manifest.set_content(pathname=manifest_file)

	return manifest

def usage():
	print "Usage: %s [-m|--manifest (file)] [-d|--search-directory (dir)] [-D|--destination (dir)] " % (sys.argv[0].split('/')[-1])
	sys.exit(1)

def main():
	import getopt

	# FLUSH STDOUT 
	sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

	search_paths = []
	destination = None
	manifests = []

	try:
		opts, args = getopt.getopt(sys.argv[1:], "D:d:m:",
			["destination=", "search-directory=", "manifest="])
	except getopt.GetoptError, err:
		print str(err)
		usage()

	for opt, arg in opts:
		if opt in [ "-D", "--destination" ]:
			destination = arg
		elif opt in [ "-d", "--search-directory" ]:
			search_paths.append(arg)
		elif opt in [ "-m", "--manifest" ]:
			try:
				manifest = load_manifest(arg)
			except IOError, err:
				print "oops, %s: %s" % (arg, str(err))
				usage()
			else:
				manifests.append(manifest)
		else:
			usage()

	if destination == None:
		usage()

	for manifest in manifests:
		mangle_paths(manifest, search_paths, destination)
		mangle_manifest(manifest)
		print manifest

	sys.exit(0)

if __name__ == "__main__":
	main()