From a3636f05135ce6f979b3665834e71bcee6f6c376 Mon Sep 17 00:00:00 2001 From: Chris Lalancette Date: Tue, 18 Oct 2011 16:53:20 -0400 Subject: [PATCH] Add a couple of patches from upstream Oz. Signed-off-by: Chris Lalancette --- oz-local-repo.patch | 216 +++++++++++++++++++++++++++++++++++++++ oz-monitor-network.patch | 97 ++++++++++++++++++ oz.spec | 14 ++- 3 files changed, 323 insertions(+), 4 deletions(-) create mode 100644 oz-local-repo.patch create mode 100644 oz-monitor-network.patch diff --git a/oz-local-repo.patch b/oz-local-repo.patch new file mode 100644 index 0000000..0041044 --- /dev/null +++ b/oz-local-repo.patch @@ -0,0 +1,216 @@ +diff -urp oz-0.7.0.orig/oz/ozutil.py oz-0.7.0/oz/ozutil.py +--- oz-0.7.0.orig/oz/ozutil.py 2011-09-09 15:01:12.000000000 -0400 ++++ oz-0.7.0/oz/ozutil.py 2011-10-03 13:36:17.485479574 -0400 +@@ -310,7 +310,8 @@ def subprocess_check_output(*popenargs, + raise SubprocessException("'%s' failed(%d): %s" % (cmd, retcode, stderr), retcode) + return (stdout, stderr, retcode) + +-def ssh_execute_command(guestaddr, sshprivkey, command, timeout=10): ++def ssh_execute_command(guestaddr, sshprivkey, command, timeout=10, ++ tunnels=None): + """ + Function to execute a command on the guest using SSH and return the output. + """ +@@ -322,14 +323,23 @@ def ssh_execute_command(guestaddr, sshpr + # + # -F /dev/null makes sure that we don't use the global or per-user + # configuration files +- return subprocess_check_output(["ssh", "-i", sshprivkey, +- "-F", "/dev/null", +- "-o", "ServerAliveInterval=30", +- "-o", "StrictHostKeyChecking=no", +- "-o", "ConnectTimeout=" + str(timeout), +- "-o", "UserKnownHostsFile=/dev/null", +- "-o", "PasswordAuthentication=no", +- "root@" + guestaddr, command]) ++ ++ cmd = ["ssh", "-i", sshprivkey, ++ "-F", "/dev/null", ++ "-o", "ServerAliveInterval=30", ++ "-o", "StrictHostKeyChecking=no", ++ "-o", "ConnectTimeout=" + str(timeout), ++ "-o", "UserKnownHostsFile=/dev/null", ++ "-o", "PasswordAuthentication=no"] ++ ++ if tunnels: ++ for host in tunnels: ++ for port in tunnels[host]: ++ cmd.append("-R %s:%s:%s" % (tunnels[host][port], host, port)) ++ ++ cmd.extend( ["root@" + guestaddr, command] ) ++ ++ return subprocess_check_output(cmd) + + def scp_copy_file(guestaddr, sshprivkey, file_to_upload, destination, + timeout=10): +Only in oz-0.7.0/oz: ozutil.py.orig +diff -urp oz-0.7.0.orig/oz/RedHat.py oz-0.7.0/oz/RedHat.py +--- oz-0.7.0.orig/oz/RedHat.py 2011-09-12 08:52:17.000000000 -0400 ++++ oz-0.7.0/oz/RedHat.py 2011-10-03 13:36:17.485479574 -0400 +@@ -26,6 +26,7 @@ import libvirt + import ConfigParser + import gzip + import guestfs ++import pycurl + + import oz.Guest + import oz.ozutil +@@ -73,6 +74,10 @@ Subsystem sftp /usr/libexec/openssh/sftp + self.tdl.name + "-ramdisk") + self.cmdline = "method=" + self.url + " ks=file:/ks.cfg" + ++ # two layer dict to track required tunnels ++ # self.tunnels[hostname][port] ++ self.tunnels = {} ++ + def _generate_new_iso(self): + """ + Method to create a new ISO based on the modified CD/DVD. +@@ -455,12 +460,13 @@ Subsystem sftp /usr/libexec/openssh/sftp + finally: + self._guestfs_handle_cleanup(g_handle) + +- def guest_execute_command(self, guestaddr, command, timeout=10): ++ def guest_execute_command(self, guestaddr, command, timeout=10, ++ tunnels=None): + """ + Method to execute a command on the guest and return the output. + """ + return oz.ozutil.ssh_execute_command(guestaddr, self.sshprivkey, +- command, timeout) ++ command, timeout, tunnels) + + def do_icicle(self, guestaddr): + """ +@@ -822,19 +828,119 @@ class RedHatCDYumGuest(RedHatCDGuest): + + return url + ++ protocol_to_default_port = { ++ 'http': '80', ++ 'https': '443', ++ 'ftp': '21', ++ } ++ ++ def _deconstruct_repo_url(self, repourl): ++ """ ++ Method to extract the protocol, port and other details from a repo URL ++ returns tuple: (protocol, hostname, port, path) ++ """ ++ # TODO: Make an offering to the regex gods to simplify this ++ url_regex = r"^(.*)(://)([^/:]+)(:)([0-9]+)([/]+)(.*)$|^(.*)(://)([^/:]+)([/]+)(.*)$" ++ ++ sr = re.search(url_regex, repourl) ++ if sr: ++ if sr.group(1): ++ # URL with port in it ++ (protocol, hostname, port, path) = sr.group(1, 3, 5, 7) ++ else: ++ # URL without port ++ (protocol, hostname, path) = sr.group(8, 10, 12) ++ port = self.protocol_to_default_port[protocol] ++ return (protocol, hostname, port, path) ++ else: ++ raise oz.OzException.OzException("Could not decode URL (%s) for port forwarding" % (repourl)) ++ ++ def _discover_repo_locality(self, repo_url, guestaddr): ++ # this is the path to the metadata XML ++ full_url = repo_url + "/repodata/repomd.xml" ++ ++ # first, check if we can access it from the host ++ self.data = '' ++ def _writefunc(buf): ++ self.data += buf ++ ++ c = pycurl.Curl() ++ c.setopt(c.URL, full_url) ++ c.setopt(c.CONNECTTIMEOUT, 5) ++ c.setopt(c.WRITEFUNCTION, _writefunc) ++ try: ++ c.perform() ++ # if we reach here, then the perform succeeded, which means we ++ # could reach the repo from the host ++ host = True ++ except pycurl.error: ++ # if we got an exception, then we could not reach the repo from ++ # the host ++ host = False ++ c.close() ++ ++ # now check if we can access it remotely ++ try: ++ self.guest_execute_command(guestaddr, "curl " + full_url) ++ # if we reach here, then the perform succeeded, which means we ++ # could reach the repo from the guest ++ guest = True ++ except oz.ozutil.SubprocessException: ++ # if we got an exception, then we could not reach the repo from ++ # the guest ++ guest = False ++ ++ return host, guest ++ + def _customize_repos(self, guestaddr): + """ + Method to generate and upload custom repository files based on the TDL. + """ ++ ++ # Starting point for our tunnels - this is the port used on our ++ # remote instance ++ tunport = 50000 ++ + self.log.debug("Installing additional repository files") + for repo in self.tdl.repositories.values(): ++ # here we go through a repo and check if it is accessible from the ++ # host and/or the guest. If a repository is available from the ++ # guest, we use the repository directly from the guest. If the ++ # repository is *only* available from the host, then we tunnel it ++ # through to the guest. If it is available from neither, we raise ++ # an exception ++ host, guest = self._discover_repo_locality(repo.url, guestaddr) ++ if not host and not guest: ++ raise oz.OzException.OzException("Could not reach repository %s from the host or the guest, aborting" % (repo.url)) ++ + filename = repo.name + ".repo" + localname = os.path.join(self.icicle_tmp, filename) + f = open(localname, 'w') + f.write("[%s]\n" % repo.name) + f.write("name=%s\n" % repo.name) +- f.write("baseurl=%s\n" % repo.url) ++ if host and not guest: ++ remote_tun_port = tunport ++ (protocol, hostname, port, path) = self._deconstruct_repo_url(repo.url) ++ if (hostname in self.tunnels) and (port in self.tunnels[hostname]): ++ # We are already tunneling this hostname and port - use the ++ # existing one ++ remote_tun_port = self.tunnels[hostname][port] ++ else: ++ # New tunnel required ++ if not (hostname in self.tunnels): ++ self.tunnels[hostname] = {} ++ self.tunnels[hostname][port] = str(remote_tun_port) ++ tunport = tunport + 1 ++ remote_url = "%s://localhost:%s/%s" % (protocol, ++ remote_tun_port, path) ++ f.write("# This is a tunneled version of local repo: (%s)\n" % (repo.url)) ++ f.write("baseurl=%s\n" % remote_url) ++ else: ++ f.write("baseurl=%s\n" % repo.url) ++ ++ f.write("skip_if_unavailable=1\n") + f.write("enabled=1\n") ++ + if repo.signed: + f.write("gpgcheck=1\n") + else: +@@ -859,7 +965,8 @@ class RedHatCDYumGuest(RedHatCDGuest): + + if packstr != '': + self.guest_execute_command(guestaddr, +- 'yum -y install %s' % (packstr)) ++ 'yum -y install %s' % (packstr), ++ tunnels=self.tunnels) + + self._customize_files(guestaddr) + +Only in oz-0.7.0/oz: RedHat.py.orig diff --git a/oz-monitor-network.patch b/oz-monitor-network.patch new file mode 100644 index 0000000..778ca62 --- /dev/null +++ b/oz-monitor-network.patch @@ -0,0 +1,97 @@ +diff -urp oz-0.7.0.orig/oz/Guest.py oz-0.7.0/oz/Guest.py +--- oz-0.7.0.orig/oz/Guest.py 2011-09-12 08:52:17.000000000 -0400 ++++ oz-0.7.0/oz/Guest.py 2011-10-05 11:31:10.518947424 -0400 +@@ -472,21 +472,42 @@ class Guest(object): + """ + # first find the disk device we are installing to; this will be + # monitored for activity during the installation +- disktarget = libxml2.parseDoc(libvirt_dom.XMLDesc(0)).xpathEval("/domain/devices/disk[@device='disk']/target") +- if len(disktarget) < 1: ++ doc = libxml2.parseDoc(libvirt_dom.XMLDesc(0)) ++ disktargets = doc.xpathEval("/domain/devices/disk/target") ++ if len(disktargets) < 1: + raise oz.OzException.OzException("Could not find disk target") +- diskdev = disktarget[0].prop('dev') +- if diskdev is None: ++ diskdevs = [] ++ for target in disktargets: ++ diskdevs.append(target.prop('dev')) ++ if not diskdevs: + raise oz.OzException.OzException("Could not find disk target device") ++ inttargets = doc.xpathEval("/domain/devices/interface/target") ++ if len(inttargets) < 1: ++ raise oz.OzException.OzException("Could not find interface target") ++ intdevs = [] ++ for target in inttargets: ++ intdevs.append(target.prop('dev')) ++ if not intdevs: ++ raise oz.OzException.OzException("Could not find interface target device") + + last_disk_activity = 0 ++ last_network_activity = 0 + inactivity_countdown = inactivity_timeout + origcount = count + while count > 0: + if count % 10 == 0: + self.log.debug("Waiting for %s to finish installing, %d/%d" % (self.tdl.name, count, origcount)) + try: +- rd_req, rd_bytes, wr_req, wr_bytes, errs = libvirt_dom.blockStats(diskdev) ++ total_disk_req = 0 ++ for dev in diskdevs: ++ rd_req, rd_bytes, wr_req, wr_bytes, errs = libvirt_dom.blockStats(dev) ++ total_disk_req += rd_req + wr_req ++ ++ total_net_bytes = 0 ++ for dev in intdevs: ++ rx_bytes, rx_packets, rx_errs, rx_drop, tx_bytes, tx_packets, tx_errs, tx_drop = libvirt_dom.interfaceStats(dev) ++ total_net_bytes += rx_bytes + tx_bytes ++ + except libvirt.libvirtError, e: + if e.get_error_domain() == libvirt.VIR_FROM_QEMU and (e.get_error_code() in [libvirt.VIR_ERR_NO_DOMAIN, libvirt.VIR_ERR_SYSTEM_ERROR, libvirt.VIR_ERR_OPERATION_FAILED]): + break +@@ -503,8 +524,8 @@ class Guest(object): + self.log.debug(" int2 is %d" % e.get_int2()) + raise + +- # if we saw no disk activity in the countdown window, we presume the +- # install has hung. Fail here ++ # if we saw no disk or network activity in the countdown window, ++ # we presume the install has hung. Fail here + if inactivity_countdown == 0: + screenshot_path = self._capture_screenshot(libvirt_dom.XMLDesc(0)) + exc_str = "No disk activity in %d seconds, failing. " % (inactivity_timeout) +@@ -514,7 +535,22 @@ class Guest(object): + exc_str += "Failed to take screenshot" + raise oz.OzException.OzException(exc_str) + +- if (rd_req + wr_req) == last_disk_activity: ++ # rd_req and wr_req are the *total* number of disk read requests and ++ # write requests ever made for this domain. Similarly rd_bytes and ++ # wr_bytes are the total number of network bytes read or written ++ # for this domain ++ ++ # we define activity as having done a read or write request on the ++ # install disk, or having done at least 4KB of network transfers in ++ # the last second. The thinking is that if the installer is putting ++ # bits on disk, there will be disk activity, so we should keep ++ # waiting. On the other hand, the installer might be downloading ++ # bits to eventually install on disk, so we look for network ++ # activity as well. We say that transfers of at least 4KB must be ++ # made, however, to try to reduce false positives from things like ++ # ARP requests ++ ++ if (total_disk_req == last_disk_activity) and (total_net_bytes < (last_network_activity + 4096)): + # if we saw no read or write requests since the last iteration, + # decrement our activity timer + inactivity_countdown -= 1 +@@ -522,7 +558,8 @@ class Guest(object): + # if we did see some activity, then we can reset the timer + inactivity_countdown = inactivity_timeout + +- last_disk_activity = rd_req + wr_req ++ last_disk_activity = total_disk_req ++ last_network_activity = total_net_bytes + count -= 1 + time.sleep(1) + +Only in oz-0.7.0/oz: Guest.py.orig diff --git a/oz.spec b/oz.spec index 7d93755..b337620 100644 --- a/oz.spec +++ b/oz.spec @@ -1,11 +1,13 @@ Summary: Library and utilities for automated guest OS installs Name: oz Version: 0.7.0 -Release: 1%{?dist} +Release: 3%{?dist} License: LGPLv2 Group: Development/Libraries URL: http://aeolusproject.org/oz.html -Source0: http://repos.fedorapeople.org/repos/aeolus/%{name}/%{version}/tarball/%{name}-%{version}.tar.gz +Source0: http://repos.fedorapeople.org/repos/aeolus/oz/%{version}/tarball/%{name}-%{version}.tar.gz +Patch1: oz-local-repo.patch +Patch2: oz-monitor-network.patch BuildArch: noarch Requires: python >= 2.5 Requires: gvnc-tools @@ -35,6 +37,9 @@ installations, with minimal input from the user. %prep %setup -q +%patch1 -p1 +%patch2 -p1 + %build python setup.py build @@ -47,7 +52,7 @@ mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/oz/isos/ mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/oz/floppycontent/ mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/oz/floppies/ mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/oz/icicletmp/ -mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/oz/jeos +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/oz/jeos/ mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/oz/kernels/ mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/oz @@ -58,6 +63,7 @@ if [ ! -f %{_sysconfdir}/oz/id_rsa-icicle-gen ]; then ssh-keygen -t rsa -b 2048 -N "" -f %{_sysconfdir}/oz/id_rsa-icicle-gen >& /dev/null fi + %files %doc README COPYING examples %dir %attr(0755, root, root) %{_sysconfdir}/oz/ @@ -68,7 +74,7 @@ fi %dir %attr(0755, root, root) %{_localstatedir}/lib/oz/floppycontent/ %dir %attr(0755, root, root) %{_localstatedir}/lib/oz/floppies/ %dir %attr(0755, root, root) %{_localstatedir}/lib/oz/icicletmp/ -%dir %attr(0755, root, root) %{_localstatedir}/lib/oz/jeos +%dir %attr(0755, root, root) %{_localstatedir}/lib/oz/jeos/ %dir %attr(0755, root, root) %{_localstatedir}/lib/oz/kernels/ %{python_sitelib}/oz %{_bindir}/oz-install