From 63d4d461fc2047b9b2b41ad4f3e5262322584473 Mon Sep 17 00:00:00 2001 From: Neal Gompa Date: Tue, 13 Nov 2018 09:59:03 -0500 Subject: [PATCH] Port to Python 3 - Backport xz multi-threading support - Refresh nss libs hack patch --- ...lti-thread-support-in-xz-compression.patch | 36 ++ ...ate-Replace-urlgrabber-with-progress.patch | 54 ++ ...-the-chroot-to-avoid-install_root-ke.patch | 44 ++ ...everything-to-be-Python-3-compatible.patch | 601 ++++++++++++++++++ appliance-tools.spec | 32 +- lookup-fake-user-for-nss.patch | 29 - 6 files changed, 755 insertions(+), 41 deletions(-) create mode 100644 0001-Enable-multi-thread-support-in-xz-compression.patch create mode 100644 0001-appcreate-Replace-urlgrabber-with-progress.patch create mode 100644 0001-open-nss-libs-in-the-chroot-to-avoid-install_root-ke.patch create mode 100644 0002-Port-everything-to-be-Python-3-compatible.patch delete mode 100644 lookup-fake-user-for-nss.patch diff --git a/0001-Enable-multi-thread-support-in-xz-compression.patch b/0001-Enable-multi-thread-support-in-xz-compression.patch new file mode 100644 index 0000000..f9ca487 --- /dev/null +++ b/0001-Enable-multi-thread-support-in-xz-compression.patch @@ -0,0 +1,36 @@ +From c7937b1115fb30c8ca23460c0af0c652708f2da5 Mon Sep 17 00:00:00 2001 +From: David Abdurachmanov +Date: Wed, 13 Jun 2018 13:25:49 +0200 +Subject: [PATCH] Enable multi-thread support in xz compression + +xz-utils starting 5.2.0 (2014-12-21) support multi-threaded compression. +This version was added to Fedora 22 (2015-01-07) and also available in +CentOS 7. + +-T 0 allows xz to use all available cores on the system assuming input +file is big enough and there is enough memory on the system. + +Tested on SiFive HiFive Unleashed (RISC-V, riscv64) with Fedora and it +showed significant time savings. + +Signed-off-by: David Abdurachmanov +--- + appcreate/appliance.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/appcreate/appliance.py b/appcreate/appliance.py +index 34971a3..70428b7 100644 +--- a/appcreate/appliance.py ++++ b/appcreate/appliance.py +@@ -643,7 +643,7 @@ class ApplianceImageCreator(ImageCreator): + + if self.__compress: + # Compress with xz using 16 MiB block size for seekability +- rc = subprocess.call(["xz", "-z", "--block-size=16777216", src]) ++ rc = subprocess.call(["xz", "-z", "--block-size=16777216", "-T 0", src]) + if rc == 0: + logging.debug("compression successful") + if rc != 0: +-- +2.17.2 + diff --git a/0001-appcreate-Replace-urlgrabber-with-progress.patch b/0001-appcreate-Replace-urlgrabber-with-progress.patch new file mode 100644 index 0000000..ff4cf77 --- /dev/null +++ b/0001-appcreate-Replace-urlgrabber-with-progress.patch @@ -0,0 +1,54 @@ +From 2f24d8182a78d44b3acf23573de5ea6d59ec2b4a Mon Sep 17 00:00:00 2001 +From: Neal Gompa +Date: Wed, 3 Oct 2018 21:33:42 -0400 +Subject: [PATCH 1/2] appcreate: Replace urlgrabber with progress + +This removes our only blocker to converting to Python 3. +--- + appcreate/appliance.py | 9 ++++----- + 1 file changed, 4 insertions(+), 5 deletions(-) + +diff --git a/appcreate/appliance.py b/appcreate/appliance.py +index 70428b7..87ffc9b 100644 +--- a/appcreate/appliance.py ++++ b/appcreate/appliance.py +@@ -3,6 +3,7 @@ + # + # Copyright 2007-2008, Red Hat Inc. + # Copyright 2008, Daniel P. Berrange ++# Copyright 2018, Neal Gompa + # + # This program is free software; you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by +@@ -31,7 +32,7 @@ from imgcreate.errors import * + from imgcreate.fs import * + from imgcreate.creator import * + from appcreate.partitionedfs import * +-import urlgrabber.progress as progress ++import progress.bar as progress + + class ApplianceImageCreator(ImageCreator): + """Installs a system into a file containing a partitioned disk image. +@@ -723,8 +724,7 @@ class ApplianceImageCreator(ImageCreator): + diskpath = "%s/%s-%s.%s" % (self._outdir, self.name, name, self.__disk_format) + disk_size = os.path.getsize(diskpath) + meter_ct = 0 +- meter = progress.TextMeter() +- meter.start(size=disk_size, text="Generating disk signature for %s-%s.%s" % (self.name, name, self.__disk_format)) ++ meter = progress.Bar("Generating disk signature for %s-%s.%s" % (self.name, name, self.__disk_format), max=disk_size) + xml += " \n" % (self.name, name, self.__disk_format, self.__disk_format) + + try: +@@ -743,8 +743,7 @@ class ApplianceImageCreator(ImageCreator): + m1.update(chunk) + if m2: + m2.update(chunk) +- meter.update(meter_ct) +- meter_ct = meter_ct + 65536 ++ meter.next(65536) + + sha1checksum = m1.hexdigest() + xml += """ %s\n""" % sha1checksum +-- +2.17.2 + diff --git a/0001-open-nss-libs-in-the-chroot-to-avoid-install_root-ke.patch b/0001-open-nss-libs-in-the-chroot-to-avoid-install_root-ke.patch new file mode 100644 index 0000000..1cd2cca --- /dev/null +++ b/0001-open-nss-libs-in-the-chroot-to-avoid-install_root-ke.patch @@ -0,0 +1,44 @@ +From 39195b686af9ae20f77bd4143a61089c48eab9de Mon Sep 17 00:00:00 2001 +From: Kevin Fenzi +Date: Thu, 5 Jul 2018 13:15:26 -0700 +Subject: [PATCH] open nss libs in the chroot to avoid install_root keeping + them open + +Reference: https://bugzilla.redhat.com/show_bug.cgi?id=1591804 +--- + tools/appliance-creator | 17 +++++++++++++++++ + 1 file changed, 17 insertions(+) + +diff --git a/tools/appliance-creator b/tools/appliance-creator +index 3ec657e..eef1ede 100755 +--- a/tools/appliance-creator ++++ b/tools/appliance-creator +@@ -171,8 +171,25 @@ def do_nss_sss_hack(): + del forgettable + return hack + ++# ++# https://bugzilla.redhat.com/show_bug.cgi?id=1591804 ++# Try and look up a unknown user in the chroot so it ++# Opens and uses all the nss libraries in the chroot ++# Instead of doing so in the install root which might ++# Keep the libraries open and fail the build. ++# ++def do_unknown_user_hack(): ++ import pwd as forgettable ++ try: ++ forgettable.getpwnam('fwefwkejkgre') ++ except: ++ pass ++ del forgettable ++ return ++ + if __name__ == "__main__": + hack = do_nss_sss_hack() ++ do_unknown_user_hack() + sys.exit(main()) + + +-- +2.17.2 + diff --git a/0002-Port-everything-to-be-Python-3-compatible.patch b/0002-Port-everything-to-be-Python-3-compatible.patch new file mode 100644 index 0000000..4b7578d --- /dev/null +++ b/0002-Port-everything-to-be-Python-3-compatible.patch @@ -0,0 +1,601 @@ +From e5d075fe142450f3acf9373049b346559e448370 Mon Sep 17 00:00:00 2001 +From: Neal Gompa +Date: Wed, 3 Oct 2018 22:23:20 -0400 +Subject: [PATCH 2/2] Port everything to be Python 3 compatible + +--- + Makefile | 13 +++++++++---- + appcreate/__init__.py | 1 + + appcreate/appliance.py | 31 +++++++++++++++++-------------- + appcreate/partitionedfs.py | 24 ++++++++++++++---------- + ec2convert/__init__.py | 5 +++-- + ec2convert/ec2config.py | 11 ++++++++--- + ec2convert/fs.py | 21 ++++++++++++++------- + ec2convert/rpmcheck.py | 2 ++ + tools/appliance-creator | 15 +++++++++------ + tools/ec2-converter | 18 +++++++++++------- + 10 files changed, 88 insertions(+), 53 deletions(-) + +diff --git a/Makefile b/Makefile +index 4357117..e8d9217 100644 +--- a/Makefile ++++ b/Makefile +@@ -4,13 +4,16 @@ INSTALL = /usr/bin/install -c + INSTALL_PROGRAM = ${INSTALL} + INSTALL_DATA = ${INSTALL} -m 644 + INSTALL_SCRIPT = ${INSTALL_PROGRAM} ++PYTHON = python ++PYTHON_PROGRAM = $(shell which $(PYTHON)) ++SED_PROGRAM = /usr/bin/sed + +-INSTALL_PYTHON = ${INSTALL} -m 644 ++INSTALL_PYTHON = $(INSTALL) -m 644 + define COMPILE_PYTHON +- python -c "import compileall as c; c.compile_dir('$(1)', force=1)" +- python -O -c "import compileall as c; c.compile_dir('$(1)', force=1)" ++ $(PYTHON_PROGRAM) -c "import compileall as c; c.compile_dir('$(1)', force=1)" ++ $(PYTHON_PROGRAM) -O -c "import compileall as c; c.compile_dir('$(1)', force=1)" + endef +-PYTHONDIR := $(shell python -c "import distutils.sysconfig as d; print d.get_python_lib()") ++PYTHONDIR := $(shell $(PYTHON_PROGRAM) -c "from __future__ import print_function; from distutils.sysconfig import get_python_lib; print(get_python_lib())") + + all: + +@@ -31,6 +34,8 @@ install: man + $(call COMPILE_PYTHON,$(DESTDIR)/$(PYTHONDIR)/ec2convert) + mkdir -p $(DESTDIR)/usr/share/man/man8 + $(INSTALL_DATA) -D docs/*.8 $(DESTDIR)/usr/share/man/man8 ++ $(SED_PROGRAM) -i "s:#!/usr/bin/python:#!$(PYTHON_PROGRAM):g" $(DESTDIR)/usr/bin/appliance-creator ++ $(SED_PROGRAM) -i "s:#!/usr/bin/python:#!$(PYTHON_PROGRAM):g" $(DESTDIR)/usr/bin/ec2-converter + + uninstall: + rm -f $(DESTDIR)/usr/bin/appliance-creator +diff --git a/appcreate/__init__.py b/appcreate/__init__.py +index 0c4cebf..92c5c72 100644 +--- a/appcreate/__init__.py ++++ b/appcreate/__init__.py +@@ -2,6 +2,7 @@ + # appcreate : Support for creating appliance images + # + # Copyright 2007, Red Hat Inc. ++# Copyright 2018, Neal Gompa + # + # This program is free software; you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by +diff --git a/appcreate/appliance.py b/appcreate/appliance.py +index 87ffc9b..812fbd9 100644 +--- a/appcreate/appliance.py ++++ b/appcreate/appliance.py +@@ -18,6 +18,9 @@ + # along with this program; if not, write to the Free Software + # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + ++from __future__ import print_function ++from builtins import chr ++from builtins import range + import os + import os.path + import glob +@@ -140,7 +143,7 @@ class ApplianceImageCreator(ImageCreator): + logging.debug("No --ondisk specified in partition line of ks file; assuming 'sda'") + disk = "sda" + +- size = parts[i].size * 1024L * 1024L ++ size = parts[i].size * 1024 * 1024 + + if len(disks) == 0: + disks.append({ 'name': disk, 'size': size }) +@@ -195,7 +198,7 @@ class ApplianceImageCreator(ImageCreator): + + try: + self.__instloop.mount() +- except MountError, e: ++ except MountError as e: + raise CreatorError("Failed mount disks : %s" % e) + + self._create_mkinitrd_config() +@@ -281,7 +284,7 @@ class ApplianceImageCreator(ImageCreator): + for version in kernels[kernel]: + versions.append(version) + +- if int(subprocess.Popen("ls " + self._instroot + "/boot/initramfs* | wc -l", shell=True, stdout=subprocess.PIPE).communicate()[0].strip()) > 0: ++ if int(subprocess.Popen("ls " + self._instroot + "/boot/initramfs* | wc -l", shell=True, stdout=subprocess.PIPE).communicate()[0].decode("utf-8").strip()) > 0: + initrd = "initramfs" + else: + initrd = "initrd" +@@ -341,7 +344,7 @@ class ApplianceImageCreator(ImageCreator): + for version in kernels[kernel]: + versions.append(version) + +- if int(subprocess.Popen("ls " + self._instroot + "/boot/initramfs* | wc -l", shell=True, stdout=subprocess.PIPE).communicate()[0].strip()) > 0: ++ if int(subprocess.Popen("ls " + self._instroot + "/boot/initramfs* | wc -l", shell=True, stdout=subprocess.PIPE).communicate()[0].decode("utf-8").strip()) > 0: + initrd = "initramfs" + else: + initrd = "initrd" +@@ -389,7 +392,7 @@ class ApplianceImageCreator(ImageCreator): + setup = "" + + i = 0 +- for name in self.__disks.keys(): ++ for name in list(self.__disks.keys()): + loopdev = self.__disks[name].device + setup += "device (hd%s) %s\n" % (i, loopdev) + i = i + 1 +@@ -404,7 +407,7 @@ class ApplianceImageCreator(ImageCreator): + grub = subprocess.Popen(["chroot", self._instroot, "/sbin/grub", "--batch", "--no-floppy"], + stdin=subprocess.PIPE) + +- grub.communicate(setup) ++ grub.communicate(setup.encode("utf-8")) + rc = grub.wait() + + subprocess.call(["umount", self._instroot + "/dev"]) +@@ -418,7 +421,7 @@ class ApplianceImageCreator(ImageCreator): + (bootdevnum, rootdevnum, rootdev, prefix) = self._get_grub_boot_config() + + i = 0 +- for name in self.__disks.keys(): ++ for name in list(self.__disks.keys()): + loopdev = self.__disks[name].device + i = i + 1 + +@@ -469,7 +472,7 @@ class ApplianceImageCreator(ImageCreator): + + def _install_extlinux(self): + i = 0 +- for name in self.__disks.keys(): ++ for name in list(self.__disks.keys()): + loopdev = self.__disks[name].device + i = i + 1 + +@@ -623,7 +626,7 @@ class ApplianceImageCreator(ImageCreator): + for f in os.listdir(self._outdir): + logging.debug("moving %s to %s" % (os.path.join(self._outdir, f), os.path.join(dst, f))) + shutil.move(os.path.join(self._outdir, f), os.path.join(dst, f)) +- print "Finished" ++ print("Finished") + + def _stage_final_image(self): + """Stage the final system image in _outdir. +@@ -638,7 +641,7 @@ class ApplianceImageCreator(ImageCreator): + #else move to _outdir + else: + logging.debug("moving disks to stage location") +- for name in self.__disks.keys(): ++ for name in list(self.__disks.keys()): + src = "%s/%s-%s.%s" % (self.__imgdir, self.name, name, self.__disk_format) + dst = "%s/%s-%s.%s" % (self._outdir, self.name, name, self.__disk_format) + +@@ -659,7 +662,7 @@ class ApplianceImageCreator(ImageCreator): + + def _convert_image(self): + #convert disk format +- for name in self.__disks.keys(): ++ for name in list(self.__disks.keys()): + dst = "%s/%s-%s.%s" % (self._outdir, self.name, name, self.__disk_format) + logging.debug("converting %s image to %s" % (self.__disks[name].lofile, dst)) + if self.__compress and self.__disk_format == "qcow2": +@@ -704,7 +707,7 @@ class ApplianceImageCreator(ImageCreator): + xml += " \n" + + i = 0 +- for name in self.__disks.keys(): ++ for name in list(self.__disks.keys()): + xml += " \n" % (self.name, name, self.__disk_format, chr(ord('a')+i)) + i = i + 1 + +@@ -720,7 +723,7 @@ class ApplianceImageCreator(ImageCreator): + xml += " \n" + + if self.checksum is True: +- for name in self.__disks.keys(): ++ for name in list(self.__disks.keys()): + diskpath = "%s/%s-%s.%s" % (self._outdir, self.name, name, self.__disk_format) + disk_size = os.path.getsize(diskpath) + meter_ct = 0 +@@ -753,7 +756,7 @@ class ApplianceImageCreator(ImageCreator): + xml += """ %s\n""" % sha256checksum + xml += " \n" + else: +- for name in self.__disks.keys(): ++ for name in list(self.__disks.keys()): + xml += " \n" % (self.name, name, self.__disk_format, self.__disk_format) + + xml += " \n" +diff --git a/appcreate/partitionedfs.py b/appcreate/partitionedfs.py +index 50260a2..3cc09c1 100644 +--- a/appcreate/partitionedfs.py ++++ b/appcreate/partitionedfs.py +@@ -4,6 +4,7 @@ + # Copyright 2007-2008, Red Hat Inc. + # Copyright 2008, Daniel P. Berrange + # Copyright 2008, David P. Huff ++# Copyright 2018, Neal Gompa + # + # This program is free software; you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by +@@ -18,6 +19,9 @@ + # along with this program; if not, write to the Free Software + # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + ++from __future__ import print_function ++from builtins import str ++from builtins import range + import os + import os.path + import glob +@@ -34,7 +38,7 @@ class PartitionedMount(Mount): + def __init__(self, disks, mountdir, partition_layout): + Mount.__init__(self, mountdir) + self.disks = {} +- for name in disks.keys(): ++ for name in list(disks.keys()): + self.disks[name] = { 'disk': disks[name], # Disk object + 'mapped': False, # True if kpartx mapping exists + 'numpart': 0, # Number of allocate partitions +@@ -61,7 +65,7 @@ class PartitionedMount(Mount): + + def __format_disks(self): + logging.debug("Formatting disks") +- for dev in self.disks.keys(): ++ for dev in list(self.disks.keys()): + d = self.disks[dev] + logging.debug("Initializing partition table for %s with %s layout" % (d['disk'].device, self.partition_layout)) + rc = subprocess.call(["/sbin/parted", "-s", d['disk'].device, "mklabel", "%s" % self.partition_layout]) +@@ -72,7 +76,7 @@ class PartitionedMount(Mount): + for n in range(len(self.partitions)): + p = self.partitions[n] + +- if not self.disks.has_key(p['disk']): ++ if p['disk'] not in self.disks: + raise MountError("No disk %s for partition %s" % (p['disk'], p['mountpoint'])) + + d = self.disks[p['disk']] +@@ -120,7 +124,7 @@ class PartitionedMount(Mount): + # raise MountError("Error creating partition on %s" % d['disk'].device) + + def __map_partitions(self): +- for dev in self.disks.keys(): ++ for dev in list(self.disks.keys()): + d = self.disks[dev] + if d['mapped']: + continue +@@ -128,7 +132,7 @@ class PartitionedMount(Mount): + kpartx = subprocess.Popen(["/sbin/kpartx", "-l", d['disk'].device], + stdout=subprocess.PIPE) + +- kpartxOutput = kpartx.communicate()[0].split("\n") ++ kpartxOutput = kpartx.communicate()[0].decode("utf-8").split("\n") + # Strip trailing blank + kpartxOutput = kpartxOutput[0:len(kpartxOutput)-1] + +@@ -185,7 +189,7 @@ class PartitionedMount(Mount): + + + def __unmap_partitions(self): +- for dev in self.disks.keys(): ++ for dev in list(self.disks.keys()): + d = self.disks[dev] + if not d['mapped']: + continue +@@ -213,12 +217,12 @@ class PartitionedMount(Mount): + self.mountOrder.sort() + self.unmountOrder.sort() + self.unmountOrder.reverse() +- print str(self.mountOrder) ++ print(str(self.mountOrder)) + + def cleanup(self): + Mount.cleanup(self) + self.__unmap_partitions() +- for dev in self.disks.keys(): ++ for dev in list(self.disks.keys()): + d = self.disks[dev] + try: + d['disk'].cleanup() +@@ -243,7 +247,7 @@ class PartitionedMount(Mount): + p['mount'] = None + + def mount(self): +- for dev in self.disks.keys(): ++ for dev in list(self.disks.keys()): + d = self.disks[dev] + d['disk'].create() + +@@ -298,7 +302,7 @@ class PartitionedMount(Mount): + + def __getuuid(self, partition): + devdata = subprocess.Popen(["/sbin/blkid", partition], stdout=subprocess.PIPE) +- devdataout = devdata.communicate()[0].split() ++ devdataout = devdata.communicate()[0].decode("utf-8").split() + for data in devdataout: + if data.startswith("UUID"): + UUID = data.replace('"', '') +diff --git a/ec2convert/__init__.py b/ec2convert/__init__.py +index 9c9d9a3..c4743c5 100644 +--- a/ec2convert/__init__.py ++++ b/ec2convert/__init__.py +@@ -15,5 +15,6 @@ + # MA 02110-1301 USA. + + +-from fs import * +-from ec2config import * +\ No newline at end of file ++from __future__ import absolute_import ++from .fs import * ++from .ec2config import * +diff --git a/ec2convert/ec2config.py b/ec2convert/ec2config.py +index 2fa47f8..09a11d2 100644 +--- a/ec2convert/ec2config.py ++++ b/ec2convert/ec2config.py +@@ -2,6 +2,8 @@ + # + # Copyright 2008, Red Hat Inc. + # Joseph Boggs ++# Copyright 2018, Neal Gompa ++# + # This program is free software; you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by + # the Free Software Foundation; version 2 of the License. +@@ -15,6 +17,7 @@ + # along with this program; if not, write to the Free Software + # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + ++from builtins import object + import os + import sys + import logging +@@ -23,7 +26,7 @@ import shutil + import ec2convert.rpmcheck as rpmcheck + import ec2convert.fs as fs + +-class EC2Config(): ++class EC2Config(object): + + def makedev(self,tmpdir): + os.popen("/sbin/MAKEDEV -d %s/dev -x console" % tmpdir) +@@ -81,7 +84,8 @@ class EC2Config(): + try: + sshdconfig_path = tmpdir + "/etc/ssh/sshd_config" + sshdconfig = open(sshdconfig_path,"w") +- except IOError, (errno, strerror): ++ except IOError as xxx_todo_changeme: ++ (errno, strerror) = xxx_todo_changeme.args + logging.error( "%s, %s" % (strerror,sshdconfig_path)) + logging.error( "The openssh_server package must be installed to convert and function properly on EC2" ) + return False +@@ -99,7 +103,8 @@ class EC2Config(): + eth0_path = tmpdir + "/etc/sysconfig/network-scripts/ifcfg-eth0" + os.system("touch %s" % eth0_path) + eth0 = open(eth0_path, "w") +- except IOError, (errno, strerror): ++ except IOError as xxx_todo_changeme1: ++ (errno, strerror) = xxx_todo_changeme1.args + logging.info( "%s, %s" % (strerror,eth0_path) ) + return False + else: +diff --git a/ec2convert/fs.py b/ec2convert/fs.py +index 105022a..9178fce 100644 +--- a/ec2convert/fs.py ++++ b/ec2convert/fs.py +@@ -2,6 +2,8 @@ + # + # Copyright 2008, Red Hat Inc. + # Joseph Boggs ++# Copyright 2018, Neal Gompa ++# + # This program is free software; you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by + # the Free Software Foundation; version 2 of the License. +@@ -15,6 +17,11 @@ + # along with this program; if not, write to the Free Software + # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + ++from __future__ import division ++ ++from builtins import str ++from builtins import object ++from past.utils import old_div + import os + import subprocess + import logging +@@ -22,7 +29,7 @@ import sys + import shutil + import time + +-class LoopBackDiskImage(): ++class LoopBackDiskImage(object): + + def setup_fs(self,imagefile,tmpdir): + loop_devices = [] +@@ -52,12 +59,12 @@ class LoopBackDiskImage(): + loop_partition_dict[dev] = label + logging.debug( dev + " : " + label) + +- dev = loop_partition_dict.values() ++ dev = list(loop_partition_dict.values()) + dev.sort() + os.mkdir(tmproot) + + for value in dev: +- for key in loop_partition_dict.keys(): ++ for key in list(loop_partition_dict.keys()): + if (value == loop_partition_dict[key]): + ld = os.popen("/sbin/losetup -f") + loop_partition_device = ld.read().strip() +@@ -67,7 +74,7 @@ class LoopBackDiskImage(): + os.system("mount -o loop /dev/mapper/%s %s%s" % (key,tmproot,value)) + + tmp_disk_space = os.popen("du -s %s|awk {'print $1'}" % tmproot) +- tmp_disk_space= int(tmp_disk_space.read()) / 1024 ++ tmp_disk_space= old_div(int(tmp_disk_space.read()), 1024) + logging.info("Disk Space Required: %sM" % str(tmp_disk_space)) + + new_disk_space = int(tmp_disk_space + ((tmp_disk_space * 0.30) + 150)) +@@ -103,7 +110,7 @@ class LoopBackDiskImage(): + os.system("rm -rf %s/*" % tmpdir) + return + +-class DirectoryImage(): ++class DirectoryImage(object): + + def setup_fs(self,imagefile,tmpdir): + tmproot = tmpdir + "-tmproot" +@@ -111,7 +118,7 @@ class DirectoryImage(): + + logging.info("TMPDIR: " + tmpdir) + tmp_disk_space = os.popen("du -s %s|awk {'print $1'}" % imagefile) +- tmp_disk_space= int(tmp_disk_space.read()) / 1024 ++ tmp_disk_space= old_div(int(tmp_disk_space.read()), 1024) + logging.info("Disk Space Required: %sM" % str(tmp_disk_space)) + + new_disk_space = int(tmp_disk_space + ((tmp_disk_space * 0.30) + 150)) +@@ -143,7 +150,7 @@ class DirectoryImage(): + return + + +-class LoopbackFSImage(): ++class LoopbackFSImage(object): + + def setup_fs(self,imagefile,tmpdir): + logging.debug("Mounting %s to %s" % (imagefile,tmpdir)) +diff --git a/ec2convert/rpmcheck.py b/ec2convert/rpmcheck.py +index 80d18f9..747b13e 100644 +--- a/ec2convert/rpmcheck.py ++++ b/ec2convert/rpmcheck.py +@@ -2,6 +2,8 @@ + # + # Copyright 2008, Red Hat Inc. + # Joseph Boggs ++# Copyright 2018, Neal Gompa ++# + # This program is free software; you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by + # the Free Software Foundation; version 2 of the License. +diff --git a/tools/appliance-creator b/tools/appliance-creator +index 3ffc22c..3ec657e 100755 +--- a/tools/appliance-creator ++++ b/tools/appliance-creator +@@ -1,8 +1,9 @@ +-#!/usr/bin/python -tt ++#!/usr/bin/python + # + # appliance-creator: Create a virtual appliance partitioned disk image + # + # Copyright 2007, Red Hat Inc. ++# Copyright 2018, Neal Gompa + # + # This program is free software; you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by +@@ -17,6 +18,7 @@ + # along with this program; if not, write to the Free Software + # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + ++from __future__ import print_function + import os + import sys + import shutil +@@ -106,7 +108,8 @@ def parse_options(args): + def main(): + try: + options = parse_options(sys.argv[1:]) +- except Usage, (msg, no_error): ++ except Usage as opterr: ++ (msg, no_error) = opterr.args + if no_error: + out = sys.stdout + ret = 0 +@@ -114,16 +117,16 @@ def main(): + out = sys.stderr + ret = 2 + if msg: +- print >> out, msg ++ print(msg, file=out) + return ret + + if os.geteuid () != 0: +- print >> sys.stderr, "You must run appliance-creator as root" ++ print("You must run appliance-creator as root", file=sys.stderr) + return 1 + + try: + ks = imgcreate.read_kickstart(options.kscfg) +- except imgcreate.CreatorError, e: ++ except imgcreate.CreatorError as e: + logging.error("Unable to load kickstart file '%s' : %s" % (options.kscfg, e)) + return 1 + +@@ -152,7 +155,7 @@ def main(): + creator.configure() + creator.unmount() + creator.package(destdir,options.package,options.include) +- except imgcreate.CreatorError, e: ++ except imgcreate.CreatorError as e: + logging.error("Unable to create appliance : %s" % e) + creator.cleanup() + return 1 +diff --git a/tools/ec2-converter b/tools/ec2-converter +index 7cfc15c..3c0029e 100755 +--- a/tools/ec2-converter ++++ b/tools/ec2-converter +@@ -1,9 +1,11 @@ +-#!/usr/bin/python -tt ++#!/usr/bin/python + # + # ec2-converter: Convert a virtual appliance image in an EC2 AMI + # + # Copyright 2008, Red Hat Inc. + # Joseph Boggs ++# Copyright 2018, Neal Gompa ++# + # This program is free software; you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by + # the Free Software Foundation; version 2 of the License. +@@ -17,6 +19,7 @@ + # along with this program; if not, write to the Free Software + # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + ++from __future__ import print_function + import os + import sys + import optparse +@@ -65,7 +68,8 @@ def parse_options(args): + def main(): + try: + options = parse_options(sys.argv[1:]) +- except Usage, (msg, no_error): ++ except Usage as opterr: ++ (msg, no_error) = opterr.args + if no_error: + out = sys.stdout + ret = 0 +@@ -73,7 +77,7 @@ def main(): + out = sys.stderr + ret = 2 + if msg: +- print >> out, msg ++ print(msg, file=out) + return ret + + if os.geteuid () != 0: +@@ -101,12 +105,12 @@ def main(): + options.tmpdir, rpmcheck, sshconfig, imagename) + + if success: +- print >> sys.stdout, "\n\nEC2 Image created as %s" % imagename +- print >> sys.stdout, "\n\nYou now need to bundle and upload the image to EC2 using the EC2 tools" +- print >> sys.stdout, "Available at: http://s3.amazonaws.com/ec2-downloads/ec2-ami-tools.noarch.rpm\n" ++ print("\n\nEC2 Image created as %s" % imagename, file=sys.stdout) ++ print("\n\nYou now need to bundle and upload the image to EC2 using the EC2 tools", file=sys.stdout) ++ print("Available at: http://s3.amazonaws.com/ec2-downloads/ec2-ami-tools.noarch.rpm\n", file=sys.stdout) + return 0 + else: +- print >> sys.stdout, "\nConversion failed" ++ print("\nConversion failed", file=sys.stdout) + return 1 + + if __name__ == "__main__": +-- +2.17.2 + diff --git a/appliance-tools.spec b/appliance-tools.spec index ac9778a..bc305b6 100644 --- a/appliance-tools.spec +++ b/appliance-tools.spec @@ -1,7 +1,7 @@ Name: appliance-tools Summary: Tools for building Appliances Version: 008.0 -Release: 10%{?dist} +Release: 11%{?dist} License: GPLv2 Group: System Environment/Base URL: https://pagure.io/appliance-tools @@ -13,20 +13,23 @@ Patch0: 0001-Set-releasever.patch Patch1: 0002-Make-it-possible-to-disable-compression.patch Patch3: 0001-Use-block-size-with-xz-to-make-seekable-xz-compresse.patch Patch4: 0001-Remove-usage-of-kickstart.get_modules-rhbz-1544075.patch -Patch5: lookup-fake-user-for-nss.patch +Patch5: 0001-open-nss-libs-in-the-chroot-to-avoid-install_root-ke.patch +Patch6: 0001-Enable-multi-thread-support-in-xz-compression.patch +Patch7: 0001-appcreate-Replace-urlgrabber-with-progress.patch +Patch8: 0002-Port-everything-to-be-Python-3-compatible.patch # Ensure system deps are installed (rhbz#1409536) -Requires: python2-imgcreate >= 1:25.0-2 -Requires: python2-urlgrabber +Requires: python3-imgcreate >= 1:25.0-2 +Requires: python3-progress Requires: curl rsync kpartx Requires: zlib Requires: qemu-img Requires: xz Requires: xfsprogs Requires: sssd-client -BuildRequires: python2-devel -BuildRequires: python-unversioned-command +BuildRequires: python3-devel BuildRequires: /usr/bin/pod2man +BuildRequires: /usr/bin/which BuildArch: noarch @@ -38,10 +41,10 @@ derived distributions such as RHEL, CentOS and others. %autosetup -p1 %build -make +# Nothing to do %install -%make_install +%make_install PYTHON=python3 # Removing license as we'll mark it as license file later rm -fv %{buildroot}%{_pkgdocdir}/COPYING @@ -53,12 +56,17 @@ rm -fv %{buildroot}%{_pkgdocdir}/COPYING %{_mandir}/man*/* %{_bindir}/appliance-creator %{_bindir}/ec2-converter -%dir %{python2_sitelib}/appcreate -%dir %{python2_sitelib}/ec2convert -%{python2_sitelib}/appcreate/* -%{python2_sitelib}/ec2convert/* +%dir %{python3_sitelib}/appcreate +%dir %{python3_sitelib}/ec2convert +%{python3_sitelib}/appcreate/* +%{python3_sitelib}/ec2convert/* %changelog +* Tue Nov 13 2018 Neal Gompa - 008.0-11 +- Port to Python 3 +- Backport xz multi-threading support +- Refresh nss libs hack patch + * Thu Jul 12 2018 Fedora Release Engineering - 008.0-10 - Rebuilt for https://fedoraproject.org/wiki/Fedora_29_Mass_Rebuild diff --git a/lookup-fake-user-for-nss.patch b/lookup-fake-user-for-nss.patch deleted file mode 100644 index eeb0e06..0000000 --- a/lookup-fake-user-for-nss.patch +++ /dev/null @@ -1,29 +0,0 @@ -diff -Nur appliance-tools-008.0.orig/tools/appliance-creator appliance-tools-008.0/tools/appliance-creator ---- appliance-tools-008.0.orig/tools/appliance-creator 2018-07-05 13:09:19.391655392 -0700 -+++ appliance-tools-008.0/tools/appliance-creator 2018-07-05 13:12:11.185416692 -0700 -@@ -168,8 +168,25 @@ - del forgettable - return hack - -+# -+# https://bugzilla.redhat.com/show_bug.cgi?id=1591804 -+# Try and look up a unknown user in the chroot so it -+# Opens and uses all the nss libraries in the chroot -+# Instead of doing so in the install root which might -+# Keep the libraries open and fail the build. -+# -+def do_unknown_user_hack(): -+ import pwd as forgettable -+ try: -+ forgettable.getpwnam('fwefwkejkgre') -+ except: -+ pass -+ del forgettable -+ return -+ - if __name__ == "__main__": - hack = do_nss_sss_hack() -+ do_unknown_user_hack() - sys.exit(main()) - -