Compare commits

...

5 Commits
rawhide ... f25

Author SHA1 Message Date
Chris Lalancette c95de72822 Update to Oz 0.16.0
Signed-off-by: Chris Lalancette <clalancette@gmail.com>
2017-08-10 10:41:20 -04:00
Ian McLeod d6b96a2bf2 automatic gzip fix 2017-06-11 21:03:53 -05:00
Ian McLeod 5a106c6c93 Add missing patch files 2017-05-10 21:46:04 -05:00
Ian McLeod 24c64e3c35 Update from Chris, Patrick and Brendan to address nss issues preventing image builds in fedora koji 2017-05-10 21:44:33 -05:00
Chris Lalancette a2a251c65b Remove rawhide dependency on python-uuid.
Signed-off-by: Chris Lalancette <clalancette@gmail.com>
2017-03-02 13:40:30 +00:00
7 changed files with 325 additions and 208 deletions

1
.gitignore vendored
View File

@ -8,3 +8,4 @@
/oz-0.13.0.tar.gz
/oz-0.14.0.tar.gz
/oz-0.15.0.tar.gz
/oz-0.16.0.tar.gz

View File

@ -0,0 +1,29 @@
From cf3b26b84a186fe7f5f4ccba8daa69b38a822807 Mon Sep 17 00:00:00 2001
From: Patrick Uiterwijk <puiterwijk@redhat.com>
Date: Tue, 23 May 2017 14:57:01 +0200
Subject: [PATCH] Prevent automatic gzip decompression by unsetting
Accept-Encoding
By not setting Accept-Encoding, servers will send Content-Type: x-gzip, which prevents
urllib3 (on behalf of requests) from decompressing it.
Fixes: #237
Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>
---
oz/ozutil.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/oz/ozutil.py b/oz/ozutil.py
index 7c75dc5..039c5a5 100644
--- a/oz/ozutil.py
+++ b/oz/ozutil.py
@@ -834,7 +834,8 @@ def http_download_file(url, fd, show_progress, logger):
"""
with requests.Session() as requests_session:
requests_session.mount('file://', LocalFileAdapter())
- response = requests_session.get(url, stream=True, allow_redirects=True)
+ response = requests_session.get(url, stream=True, allow_redirects=True,
+ headers={'Accept-Encoding': ''})
file_size = int(response.headers.get('Content-Length'))
chunk_size = 10*1024*1024
done = 0

View File

@ -0,0 +1,129 @@
From 11c32a684b20b92f800d3ffa670dc8773e22bb92 Mon Sep 17 00:00:00 2001
From: Chris Lalancette <clalancette@gmail.com>
Date: Thu, 2 Mar 2017 18:52:06 -0500
Subject: [PATCH] Support file:// URLs again.
The switch to requests broke support for file:// URLs.
Re-implement it here by implementing our own Adapter for
requests which allows file:// to work again. While in
here, I also fixed another problem having to do with
how things are printed out during the download; this
should look better now.
Signed-off-by: Chris Lalancette <clalancette@gmail.com>
---
oz/ozutil.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++--------
1 file changed, 66 insertions(+), 10 deletions(-)
diff --git a/oz/ozutil.py b/oz/ozutil.py
index 2523ea7..09cd124 100644
--- a/oz/ozutil.py
+++ b/oz/ozutil.py
@@ -30,6 +30,7 @@
import time
import select
import contextlib
+import urllib
try:
import configparser
except ImportError:
@@ -744,6 +745,57 @@ def default_screenshot_dir():
"""
return os.path.join(default_data_dir(), "screenshots")
+class LocalFileAdapter(requests.adapters.BaseAdapter):
+ @staticmethod
+ def _chkpath(method, path):
+ """Return an HTTP status for the given filesystem path."""
+ if method.lower() in ('put', 'delete'):
+ return 501, "Not Implemented" # TODO
+ elif method.lower() not in ('get', 'head', 'post'):
+ return 405, "Method Not Allowed"
+ elif os.path.isdir(path):
+ return 400, "Path Not A File"
+ elif not os.path.isfile(path):
+ return 404, "File Not Found"
+ elif not os.access(path, os.R_OK):
+ return 403, "Access Denied"
+ else:
+ return 200, "OK"
+
+ def send(self, req, **kwargs): # pylint: disable=unused-argument
+ """Return the file specified by the given request
+
+ @type req: C{PreparedRequest}
+ @todo: Should I bother filling `response.headers` and processing
+ If-Modified-Since and friends using `os.stat`?
+ """
+ path = os.path.normcase(os.path.normpath(urllib.url2pathname(req.path_url)))
+ response = requests.Response()
+
+ response.status_code, response.reason = self._chkpath(req.method, path)
+ if response.status_code == 200 and req.method.lower() != 'head':
+ try:
+ response.raw = open(path, 'rb')
+ except (OSError, IOError), err:
+ response.status_code = 500
+ response.reason = str(err)
+
+ if isinstance(req.url, bytes):
+ response.url = req.url.decode('utf-8')
+ else:
+ response.url = req.url
+
+ response.headers['Content-Length'] = os.path.getsize(path)
+ response.headers['Accept-Ranges'] = 'bytes'
+ response.headers['Redirect-URL'] = req.url
+ response.request = req
+ response.connection = self
+
+ return response
+
+ def close(self):
+ pass
+
def http_get_header(url, redirect=True):
"""
Function to get the HTTP headers from a URL. The available headers will be
@@ -755,11 +807,13 @@ def http_get_header(url, redirect=True):
'Redirect-URL' will always be None in the redirect=True case, and may be
None in the redirect=True case if no redirects were required.
"""
- with contextlib.closing(requests.post(url, allow_redirects=redirect, stream=True, timeout=10)) as r:
- info = r.headers
- info['HTTP-Code'] = r.status_code
+ with requests.Session() as requests_session:
+ requests_session.mount('file://', LocalFileAdapter())
+ response = requests_session.post(url, allow_redirects=redirect, stream=True, timeout=10)
+ info = response.headers
+ info['HTTP-Code'] = response.status_code
if not redirect:
- info['Redirect-URL'] = r.headers.get('Location')
+ info['Redirect-URL'] = response.headers.get('Location')
else:
info['Redirect-URL'] = None
@@ -769,15 +823,17 @@ def http_download_file(url, fd, show_progress, logger):
"""
Function to download a file from url to file descriptor fd.
"""
- with contextlib.closing(requests.get(url, stream=True, allow_redirects=True)) as r:
- file_size = int(r.headers.get('Content-Length'))
+ with requests.Session() as requests_session:
+ requests_session.mount('file://', LocalFileAdapter())
+ response = requests_session.get(url, stream=True, allow_redirects=True)
+ file_size = int(response.headers.get('Content-Length'))
chunk_size = 10*1024*1024
- i = 0
- for chunk in r.iter_content(chunk_size):
- i = i + 1
+ done = 0
+ for chunk in response.iter_content(chunk_size):
write_bytes_to_fd(fd, chunk)
+ done += len(chunk)
if show_progress:
- logger.debug("%dkB of %dkB" % ((i * chunk_size) / 1024, file_size / 1024))
+ logger.debug("%dkB of %dkB" % (done / 1024, file_size / 1024))
def ftp_download_directory(server, username, password, basepath, destination):
"""

View File

@ -0,0 +1,146 @@
From 32a4edd3124e7acb2dac02ff77aeea944313cbc2 Mon Sep 17 00:00:00 2001
From: Patrick Uiterwijk <puiterwijk@redhat.com>
Date: Sat, 28 Jan 2017 21:33:13 +0000
Subject: [PATCH] Replace pycurl with requests
Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>
---
oz/ozutil.py | 102 +++++++++++--------------------------------------------
requirements.txt | 2 +-
4 files changed, 22 insertions(+), 86 deletions(-)
diff --git a/oz/ozutil.py b/oz/ozutil.py
index ade670f..2523ea7 100644
--- a/oz/ozutil.py
+++ b/oz/ozutil.py
@@ -1,5 +1,5 @@
# Copyright (C) 2010,2011 Chris Lalancette <clalance@redhat.com>
-# Copyright (C) 2012-2016 Chris Lalancette <clalancette@gmail.com>
+# Copyright (C) 2012-2017 Chris Lalancette <clalancette@gmail.com>
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -25,10 +25,11 @@
import errno
import stat
import shutil
-import pycurl
+import requests
import gzip
import time
import select
+import contextlib
try:
import configparser
except ImportError:
@@ -754,48 +755,13 @@ def http_get_header(url, redirect=True):
'Redirect-URL' will always be None in the redirect=True case, and may be
None in the redirect=True case if no redirects were required.
"""
- info = {}
- def _header(buf):
- """
- Internal function that is called back from pycurl perform() for
- header data.
- """
- buf = buf.strip()
- if len(buf) == 0:
- return
-
- split = buf.split(':')
- if len(split) < 2:
- # not a valid header; skip
- return
- key = split[0].strip()
- value = split[1].strip()
- info[key] = value
-
- def _data(buf):
- """
- Empty function that is called back from pycurl perform() for body data.
- """
- pass
-
- c = pycurl.Curl()
- c.setopt(c.URL, url)
- c.setopt(c.NOBODY, True)
- c.setopt(c.HEADERFUNCTION, _header)
- c.setopt(c.HEADER, True)
- c.setopt(c.WRITEFUNCTION, _data)
- if redirect:
- c.setopt(c.FOLLOWLOCATION, True)
- c.perform()
- info['HTTP-Code'] = c.getinfo(c.HTTP_CODE)
- if info['HTTP-Code'] == 0:
- # if this was a file:/// URL, then the HTTP_CODE returned 0.
- # set it to 200 to be compatible with http
- info['HTTP-Code'] = 200
- if not redirect:
- info['Redirect-URL'] = c.getinfo(c.REDIRECT_URL)
-
- c.close()
+ with contextlib.closing(requests.post(url, allow_redirects=redirect, stream=True, timeout=10)) as r:
+ info = r.headers
+ info['HTTP-Code'] = r.status_code
+ if not redirect:
+ info['Redirect-URL'] = r.headers.get('Location')
+ else:
+ info['Redirect-URL'] = None
return info
@@ -803,45 +769,15 @@ def http_download_file(url, fd, show_progress, logger):
"""
Function to download a file from url to file descriptor fd.
"""
- class Progress(object):
- """
- Internal class to represent progress on the connection. This is only
- required so that we have somewhere to store the "last_mb" variable
- that is not global.
- """
- def __init__(self):
- self.last_mb = -1
-
- def progress(self, down_total, down_current, up_total, up_current):
- """
- Function that is called back from the pycurl perform() method to
- update the progress information.
- """
- if down_total == 0:
- return
- current_mb = int(down_current) / 10485760
- if current_mb > self.last_mb or down_current == down_total:
- self.last_mb = current_mb
- logger.debug("%dkB of %dkB" % (down_current/1024, down_total/1024))
-
- def _data(buf):
- """
- Function that is called back from the pycurl perform() method to
- actually write data to disk.
- """
- write_bytes_to_fd(fd, buf)
-
- progress = Progress()
- c = pycurl.Curl()
- c.setopt(c.URL, url)
- c.setopt(c.CONNECTTIMEOUT, 5)
- c.setopt(c.WRITEFUNCTION, _data)
- c.setopt(c.FOLLOWLOCATION, 1)
- if show_progress:
- c.setopt(c.NOPROGRESS, 0)
- c.setopt(c.PROGRESSFUNCTION, progress.progress)
- c.perform()
- c.close()
+ with contextlib.closing(requests.get(url, stream=True, allow_redirects=True)) as r:
+ file_size = int(r.headers.get('Content-Length'))
+ chunk_size = 10*1024*1024
+ i = 0
+ for chunk in r.iter_content(chunk_size):
+ i = i + 1
+ write_bytes_to_fd(fd, chunk)
+ if show_progress:
+ logger.debug("%dkB of %dkB" % ((i * chunk_size) / 1024, file_size / 1024))
def ftp_download_directory(server, username, password, basepath, destination):
"""

View File

@ -1,200 +0,0 @@
diff --git a/oz.cfg b/oz.cfg
index 5be1cb0..8913f9d 100644
--- a/oz.cfg
+++ b/oz.cfg
@@ -19,3 +19,9 @@ jeos = no
[icicle]
safe_generation = no
+
+[timeouts]
+install = 1200
+inactivity = 300
+boot = 300
+shutdown = 90
diff --git a/oz/Guest.py b/oz/Guest.py
index 54a0e8c..22c93c4 100644
--- a/oz/Guest.py
+++ b/oz/Guest.py
@@ -203,6 +203,16 @@ class Guest(object):
'safe_generation',
False)
+ # configuration of 'timeouts' section
+ self.default_install_timeout = int(oz.ozutil.config_get_key(config, 'timeouts',
+ 'install', 1200))
+ self.inactivity_timeout = int(oz.ozutil.config_get_key(config, 'timeouts',
+ 'inactivity', 300))
+ self.boot_timeout = int(oz.ozutil.config_get_key(config, 'timeouts',
+ 'boot', 300))
+ self.shutdown_timeout = int(oz.ozutil.config_get_key(config, 'timeouts',
+ 'shutdown', 90))
+
# only pull a cached JEOS if it was built with the correct image type
jeos_extension = self.image_type
if self.image_type == 'raw':
@@ -471,6 +481,10 @@ class Guest(object):
if self.tdl.arch == "armv7l":
cmdline += " console=ttyAMA0"
self.lxml_subelement(osNode, "cmdline", cmdline)
+ if self.tdl.arch == "aarch64":
+ loader,nvram = oz.ozutil.find_uefi_firmware(self.tdl.arch)
+ self.lxml_subelement(osNode, "loader", loader, {'readonly': 'yes', 'type': 'pflash'})
+ self.lxml_subelement(osNode, "nvram", None, {'template': nvram})
# poweroff, reboot, crash
self.lxml_subelement(domain, "on_poweroff", "destroy")
self.lxml_subelement(domain, "on_reboot", "destroy")
@@ -761,8 +775,7 @@ class Guest(object):
# the passed in exception was None, just raise a generic error
raise oz.OzException.OzException("Unknown libvirt error")
- def _wait_for_install_finish(self, libvirt_dom, count,
- inactivity_timeout=300):
+ def _wait_for_install_finish(self, libvirt_dom, count):
"""
Method to wait for an installation to finish. This will wait around
until either the VM has gone away (at which point it is assumed the
@@ -774,7 +787,7 @@ class Guest(object):
last_disk_activity = 0
last_network_activity = 0
- inactivity_countdown = inactivity_timeout
+ inactivity_countdown = self.inactivity_timeout
origcount = count
saved_exception = None
while count > 0 and inactivity_countdown > 0:
@@ -809,7 +822,7 @@ class Guest(object):
inactivity_countdown -= 1
else:
# if we did see some activity, then we can reset the timer
- inactivity_countdown = inactivity_timeout
+ inactivity_countdown = self.inactivity_timeout
last_disk_activity = total_disk_req
last_network_activity = total_net_bytes
@@ -826,18 +839,19 @@ class Guest(object):
# if we saw no disk or network activity in the countdown window,
# we presume the install has hung. Fail here
screenshot_text = self._capture_screenshot(libvirt_dom)
- raise oz.OzException.OzException("No disk activity in %d seconds, failing. %s" % (inactivity_timeout, screenshot_text))
+ raise oz.OzException.OzException("No disk activity in %d seconds, failing. %s" % (self.inactivity_timeout, screenshot_text))
# We get here only if we got a libvirt exception
self._wait_for_clean_shutdown(libvirt_dom, saved_exception)
self.log.info("Install of %s succeeded", self.tdl.name)
- def _wait_for_guest_shutdown(self, libvirt_dom, count=90):
+ def _wait_for_guest_shutdown(self, libvirt_dom):
"""
Method to wait around for orderly shutdown of a running guest. Returns
True if the guest shutdown in the specified time, False otherwise.
"""
+ count = self.shutdown_timeout
origcount = count
saved_exception = None
while count > 0:
@@ -1235,12 +1249,12 @@ class Guest(object):
sock.connect(('127.0.0.1', self.listen_port))
addr = None
- count = 300
+ count = self.boot_timeout
data = ''
while count > 0:
do_sleep = True
if count % 10 == 0:
- self.log.debug("Waiting for guest %s to boot, %d/300", self.tdl.name, count)
+ self.log.debug("Waiting for guest %s to boot, %d/%d", self.tdl.name, count, self.boot_timeout)
try:
# note that we have to build the data up here, since there
# is no guarantee that we will get the whole write in one go
diff --git a/oz/RedHat.py b/oz/RedHat.py
index aaa102c..896edb7 100644
--- a/oz/RedHat.py
+++ b/oz/RedHat.py
@@ -334,6 +334,11 @@ label customiso
# part 4; make sure the guest announces itself
self.log.debug("Step 4: Guest announcement")
+ if self.tdl.arch in [ 'ppc64', 'ppc64le' ]:
+ announce_device = '/dev/hvc1'
+ else:
+ announce_device = '/dev/ttyS1'
+
scriptfile = os.path.join(self.icicle_tmp, "script")
if g_handle.exists("/etc/NetworkManager/dispatcher.d"):
@@ -342,9 +347,9 @@ label customiso
#!/bin/bash
if [ "$1" = "eth0" -a "$2" = "up" ]; then
- echo -n "!$DHCP4_IP_ADDRESS,%s!" > /dev/ttyS1
+ echo -n "!$DHCP4_IP_ADDRESS,%s!" > %s
fi
-""" % (self.uuid))
+""" % (self.uuid, announce_device))
try:
g_handle.upload(scriptfile,
@@ -364,8 +369,8 @@ DEV=$(/bin/awk '{if ($2 == 0) print $1}' /proc/net/route) &&
[ -z "$DEV" ] && exit 0
ADDR=$(/sbin/ip -4 -o addr show dev $DEV | /bin/awk '{print $4}' | /bin/cut -d/ -f1) &&
[ -z "$ADDR" ] && exit 0
-echo -n "!$ADDR,%s!" > /dev/ttyS1
-""" % (self.uuid))
+echo -n "!$ADDR,%s!" > %s
+""" % (self.uuid, announce_device))
try:
g_handle.upload(scriptfile, '/root/reportip')
diff --git a/oz/ozutil.py b/oz/ozutil.py
index eb4cd6f..775576b 100644
--- a/oz/ozutil.py
+++ b/oz/ozutil.py
@@ -970,3 +970,45 @@ def recursively_add_write_bit(inputdir):
except OSError as err:
if err.errno != errno.ENOENT:
raise
+
+def find_uefi_firmware(arch):
+ # Yuck. Finding the UEFI firmware to start certain guests (like aarch64)
+ # is a really nasty process. While slightly out of date, this blog post
+ # describes the mess: http://blog.wikichoon.com/2016/01/uefi-support-in-virt-install-and-virt.html
+ # Here, we replicate what libguestfs is doing here, which is to essentially
+ # hardcode paths where UEFI firmware can be found on popular distributions.
+ # I verified that these files exist on both Fedora/RHEL and Ubuntu.
+ # Hopefully there will be a nicer way to do this in the future.
+ class UEFI(object):
+ def __init__(self, loader, nvram):
+ self.loader = loader
+ self.nvram = nvram
+
+ def exists(self):
+ if os.path.exists(self.loader) and os.path.exists(self.nvram):
+ return True
+ return False
+
+ if arch in ['i386', 'i486', 'i586', 'i686']:
+ uefi_list = [UEFI('/usr/share/edk2.git/ovmf-ia32/OVMF_CODE-pure-efi.fd',
+ '/usr/share/edk2.git/ovmf-ia32/OVMF_VARS-pure-efi.fd')]
+ elif arch in ['x86_64']:
+ uefi_list = [UEFI('/usr/share/OVMF/OVMF_CODE.fd',
+ '/usr/share/OVMF/OVMF_VARS.fd'),
+ UEFI('/usr/share/edk2.git/ovmf-x64/OVMF_CODE-pure-efi.fd',
+ '/usr/share/edk2.git/ovmf-x64/OVMF_VARS-pure-efi.fd')]
+ elif arch in ['aarch64']:
+ uefi_list = [UEFI('/usr/share/AAVMF/AAVMF_CODE.fd',
+ '/usr/share/AAVMF/AAVMF_VARS.fd'),
+ UEFI('/usr/share/edk2/aarch64/QEMU_EFI-pflash.raw',
+ '/usr/share/edk2/aarch64/vars-template-pflash.raw'),
+ UEFI('/usr/share/edk2.git/aarch64/QEMU_EFI-pflash.raw',
+ '/usr/share/edk2.git/aarch64/vars-template-pflash.raw')]
+ else:
+ raise Exception("Invalid arch for UEFI firmware")
+
+ for uefi in uefi_list:
+ if uefi.exists():
+ return uefi.loader,uefi.nvram
+
+ raise Exception("UEFI firmware is not installed!")

26
oz.spec
View File

@ -1,12 +1,11 @@
Summary: Library and utilities for automated guest OS installs
Name: oz
Version: 0.15.0
Release: 4%{?dist}
Version: 0.16.0
Release: 1%{?dist}
License: LGPLv2
Group: Development/Libraries
URL: http://github.com/clalancette/oz
Source0: http://github.com/clalancette/%{name}/files/150178/%{name}-%{version}.tar.gz
Patch0: oz-secondaryarch.patch
Source0: https://github.com/clalancette/%{name}/files/1209916/%{name}-%{version}.tar.gz
BuildArch: noarch
Requires: python >= 2.5
Requires: python-libguestfs >= 1.18
@ -19,12 +18,12 @@ Requires: libvirt-python >= 0.9.7
Requires: libvirt-daemon-kvm
Requires: libvirt-daemon-qemu
Requires: libvirt-daemon-config-network
Requires: python-pycurl
Requires: python-requests
Requires: genisoimage
Requires: mtools
Requires: python-uuid
Requires: openssh-clients
Requires: m2crypto
Requires: python-monotonic
BuildRequires: python
@ -34,7 +33,6 @@ installations, with minimal input from the user.
%prep
%setup -q
%patch0 -p1
%build
python setup.py build
@ -82,6 +80,20 @@ fi
%{_mandir}/man1/*
%changelog
* Tue Aug 08 2017 Chris Lalancette <clalancette@gmail.com> - 0.16.0-1
- Release 0.16.0
* Fri Jun 09 2017 Brendan Reilly <breilly@redhat.com> - 0.15.0-7
- Add patch to prevent automatic gzip decompression
- Pulled from upstream https://github.com/clalancette/oz/pull/238
* Wed May 10 2017 Brendan Reilly <breilly@redhat.com> - 0.15.0-6
- Add patch to use openssl instead of nss
- Pulled from upstream https://github.com/clalancette/oz/pull/231
* Thu Sep 22 2016 Chris Lalancette <clalancette@gmail.com> - 0.15.0-5
- Remove rawhide dependency on python-uuid
* Wed Aug 10 2016 Ian McLeod <imcleod@redhat.com> - 0.15.0-4
- Version bump to simplify upgrade for Fedora rel-eng

View File

@ -1 +1 @@
b83443171dc1ae47740893def6ee6367 oz-0.15.0.tar.gz
SHA512 (oz-0.16.0.tar.gz) = cff35460e0a0a231889dc15d8e68d0c86dde93a51b1f77695805470c8a6395833ebdd34c1a3983c3be23d862ff302dfafb53b6dcec0b2c02bcd325ae7bd140c9