diff --git a/0001-tests-Reduce-excessive-time-taken-by-some-tests.patch b/0001-tests-Reduce-excessive-time-taken-by-some-tests.patch new file mode 100644 index 0000000..1a94cae --- /dev/null +++ b/0001-tests-Reduce-excessive-time-taken-by-some-tests.patch @@ -0,0 +1,167 @@ +From fdbd0ef69f0c222d0cdc35d3c48297aff549abce Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 13 Dec 2019 15:38:09 +0000 +Subject: [PATCH 1/2] tests: Reduce excessive time taken by some tests. + +For context see: +https://www.redhat.com/archives/libguestfs/2019-December/msg00074.html + +This is mainly about changing tests to use libnbd/nbdsh instead of +libguestfs where libguestfs isn't really necessary for the test. +--- + tests/test-cache-on-read.sh | 37 ++++++++++++++++--------------------- + tests/test-cache.sh | 37 ++++++++++++++++++++----------------- + tests/test-cow.sh | 9 ++++++--- + 3 files changed, 42 insertions(+), 41 deletions(-) + +diff --git a/tests/test-cache-on-read.sh b/tests/test-cache-on-read.sh +index 4ecf3af..6196373 100755 +--- a/tests/test-cache-on-read.sh ++++ b/tests/test-cache-on-read.sh +@@ -30,15 +30,11 @@ + # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + +-# XXX Suggestion to improve this test: Use the delay filter below the +-# cache filter, and time reads to prove that the second read is faster +-# because it isn't going through the delay filter and plugin. +- + source ./functions.sh + set -e + set -x + +-requires guestfish --version ++requires nbdsh --version + + sock=`mktemp -u` + files="cache-on-read.img $sock cache-on-read.pid" +@@ -46,7 +42,7 @@ rm -f $files + cleanup_fn rm -f $files + + # Create an empty base image. +-truncate -s 1G cache-on-read.img ++truncate -s 128K cache-on-read.img + + # Run nbdkit with the caching filter and cache-on-read set. + start_nbdkit -P cache-on-read.pid -U $sock \ +@@ -54,19 +50,18 @@ start_nbdkit -P cache-on-read.pid -U $sock \ + file cache-on-read.img \ + cache-on-read=true + +-# Open the overlay and perform some operations. +-guestfish --format=raw -a "nbd://?socket=$sock" <<'EOF' +- run +- part-disk /dev/sda gpt +- mkfs ext4 /dev/sda1 +- mount /dev/sda1 / +- fill-dir / 10000 +- fill-pattern "abcde" 5M /large +- write /hello "hello, world" +-EOF ++nbdsh --connect "nbd+unix://?socket=$sock" \ ++ -c ' ++# Write some pattern data to the overlay and check it reads back OK. ++buf = b"abcd" * 16384 ++h.pwrite (buf, 32768) ++zero = h.pread (32768, 0) ++assert zero == bytearray (32768) ++buf2 = h.pread (65536, 32768) ++assert buf == buf2 + +-# Check the last files we created exist. +-guestfish --ro -a cache-on-read.img -m /dev/sda1 <<'EOF' +- cat /hello +- cat /large | cat >/dev/null +-EOF ++# XXX Suggestion to improve this test: Use the delay filter below the ++# cache filter, and time reads to prove that the second read is faster ++# because it is not going through the delay filter and plugin. ++# XXX second h.pread here ... ++' +diff --git a/tests/test-cache.sh b/tests/test-cache.sh +index bafada2..eead7c8 100755 +--- a/tests/test-cache.sh ++++ b/tests/test-cache.sh +@@ -34,7 +34,7 @@ source ./functions.sh + set -e + set -x + +-requires guestfish --version ++requires nbdsh --version + + sock=`mktemp -u` + files="cache.img $sock cache.pid" +@@ -42,24 +42,27 @@ rm -f $files + cleanup_fn rm -f $files + + # Create an empty base image. +-truncate -s 1G cache.img ++truncate -s 128K cache.img + + # Run nbdkit with the caching filter. + start_nbdkit -P cache.pid -U $sock --filter=cache file cache.img + +-# Open the overlay and perform some operations. +-guestfish --format=raw -a "nbd://?socket=$sock" <<'EOF' +- run +- part-disk /dev/sda gpt +- mkfs ext4 /dev/sda1 +- mount /dev/sda1 / +- fill-dir / 10000 +- fill-pattern "abcde" 5M /large +- write /hello "hello, world" +-EOF ++nbdsh --connect "nbd+unix://?socket=$sock" \ ++ -c ' ++# Write some pattern data to the overlay and check it reads back OK. ++buf = b"abcd" * 16384 ++h.pwrite (buf, 32768) ++zero = h.pread (32768, 0) ++assert zero == bytearray (32768) ++buf2 = h.pread (65536, 32768) ++assert buf == buf2 + +-# Check the last files we created exist. +-guestfish --ro -a cache.img -m /dev/sda1 <<'EOF' +- cat /hello +- cat /large | cat >/dev/null +-EOF ++# Flushing should write through to the underlying file. ++h.flush () ++ ++with open ("cache.img", "rb") as file: ++ zero = file.read (32768) ++ assert zero == bytearray (32768) ++ buf2 = file.read (65536) ++ assert buf == buf2 ++' +diff --git a/tests/test-cow.sh b/tests/test-cow.sh +index 5567b77..dea518c 100755 +--- a/tests/test-cow.sh ++++ b/tests/test-cow.sh +@@ -42,7 +42,11 @@ rm -f $files + cleanup_fn rm -f $files + + # Create a base image which is partitioned with an empty filesystem. +-guestfish -N cow-base.img=fs exit ++rm -rf cow.d ++mkdir cow.d ++cleanup_fn rm -rf cow.d ++nbdkit -fv -U - linuxdisk cow.d size=100M \ ++ --run 'qemu-img convert $nbd cow-base.img' + lastmod="$(stat -c "%y" cow-base.img)" + + # Run nbdkit with a COW overlay. +@@ -50,8 +54,7 @@ start_nbdkit -P cow.pid -U $sock --filter=cow file cow-base.img + + # Write some data into the overlay. + guestfish --format=raw -a "nbd://?socket=$sock" -m /dev/sda1 < +Date: Sat, 14 Dec 2019 12:20:55 +0000 +Subject: [PATCH 2/2] tests: Improve test-readahead test. + +See +https://www.redhat.com/archives/libguestfs/2019-December/msg00088.html +--- + tests/Makefile.am | 13 +-- + tests/test-readahead-test-plugin.sh | 48 +++++++++ + tests/test-readahead-test-request.py | 44 +++++++++ + tests/test-readahead.c | 140 --------------------------- + tests/test-readahead.sh | 67 +++++++++++++ + 5 files changed, 166 insertions(+), 146 deletions(-) + create mode 100755 tests/test-readahead-test-plugin.sh + create mode 100755 tests/test-readahead-test-request.py + delete mode 100644 tests/test-readahead.c + create mode 100755 tests/test-readahead.sh + +diff --git a/tests/Makefile.am b/tests/Makefile.am +index 1932eea..a5e9897 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -168,7 +168,10 @@ EXTRA_DIST = \ + test-rate.sh \ + test-rate-dynamic.sh \ + test.rb \ ++ test-readahead.sh \ + test-readahead-copy.sh \ ++ test-readahead-test-plugin.sh \ ++ test-readahead-test-request.py \ + test-retry.sh \ + test-retry-extents.sh \ + test-retry-size.sh \ +@@ -1050,12 +1053,10 @@ TESTS += \ + $(NULL) + + # readahead filter test. +-LIBGUESTFS_TESTS += test-readahead +-TESTS += test-readahead-copy.sh +- +-test_readahead_SOURCES = test-readahead.c test.h +-test_readahead_CFLAGS = $(WARNINGS_CFLAGS) $(LIBGUESTFS_CFLAGS) +-test_readahead_LDADD = libtest.la $(LIBGUESTFS_LIBS) ++TESTS += \ ++ test-readahead.sh \ ++ test-readahead-copy.sh \ ++ $(NULL) + + # retry filter test. + TESTS += \ +diff --git a/tests/test-readahead-test-plugin.sh b/tests/test-readahead-test-plugin.sh +new file mode 100755 +index 0000000..3c068cb +--- /dev/null ++++ b/tests/test-readahead-test-plugin.sh +@@ -0,0 +1,48 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2018-2019 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++# Test plugin used by test-readahead.sh. ++ ++case "$1" in ++ get_size) ++ # test-readahead-test-request.py will make 10 requests of 512 ++ # bytes each, so this just has to be >= 512*10. ++ echo 1M ++ ;; ++ pread) ++ sleep 5 ++ dd if=/dev/zero count=$3 iflag=count_bytes ++ ;; ++ *) ++ exit 2 ++ ;; ++esac +diff --git a/tests/test-readahead-test-request.py b/tests/test-readahead-test-request.py +new file mode 100755 +index 0000000..fd615bd +--- /dev/null ++++ b/tests/test-readahead-test-request.py +@@ -0,0 +1,44 @@ ++#!/usr/bin/env python3 ++# -*- python -*- ++# nbdkit ++# Copyright (C) 2018-2019 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++# Used by test-readahead.sh to make a linear series of requests ++# with a constant, known size. ++ ++import nbd ++import sys ++ ++h = nbd.NBD () ++h.connect_unix (sys.argv[1]) ++ ++for i in range (0, 512*10, 512): ++ h.pread (512, i) +diff --git a/tests/test-readahead.c b/tests/test-readahead.c +deleted file mode 100644 +index 1772228..0000000 +--- a/tests/test-readahead.c ++++ /dev/null +@@ -1,140 +0,0 @@ +-/* nbdkit +- * Copyright (C) 2017-2019 Red Hat Inc. +- * +- * Redistribution and use in source and binary forms, with or without +- * modification, are permitted provided that the following conditions are +- * met: +- * +- * * Redistributions of source code must retain the above copyright +- * notice, this list of conditions and the following disclaimer. +- * +- * * Redistributions in binary form must reproduce the above copyright +- * notice, this list of conditions and the following disclaimer in the +- * documentation and/or other materials provided with the distribution. +- * +- * * Neither the name of Red Hat nor the names of its contributors may be +- * used to endorse or promote products derived from this software without +- * specific prior written permission. +- * +- * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND +- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +- * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +- * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR +- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +- * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +- * SUCH DAMAGE. +- */ +- +-#include +- +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include +- +-#include "test.h" +- +-int +-main (int argc, char *argv[]) +-{ +- guestfs_h *g; +- int r; +- char *data; +- char **files; +- size_t i; +- +- /* The idea behind the test is that the readahead filter should not +- * interfere with normal, complex, non-sequential read and write +- * operations (although it will slow things down a lot). +- */ +- if (test_start_nbdkit ("--filter", "readahead", +- "memory", "1G", NULL) == -1) +- exit (EXIT_FAILURE); +- +- g = guestfs_create (); +- if (g == NULL) { +- perror ("guestfs_create"); +- exit (EXIT_FAILURE); +- } +- +- r = guestfs_add_drive_opts (g, "", +- GUESTFS_ADD_DRIVE_OPTS_FORMAT, "raw", +- GUESTFS_ADD_DRIVE_OPTS_PROTOCOL, "nbd", +- GUESTFS_ADD_DRIVE_OPTS_SERVER, server, +- -1); +- if (r == -1) +- exit (EXIT_FAILURE); +- +- if (guestfs_launch (g) == -1) +- exit (EXIT_FAILURE); +- +- if (guestfs_part_disk (g, "/dev/sda", "mbr") == -1) +- exit (EXIT_FAILURE); +- if (guestfs_mkfs (g, "ext2", "/dev/sda1") == -1) +- exit (EXIT_FAILURE); +- +- if (guestfs_mount_options (g, "discard", "/dev/sda1", "/") == -1) +- exit (EXIT_FAILURE); +- +-#define filename "/hello.txt" +-#define content "hello, people of the world" +- +- if (guestfs_write (g, filename, content, strlen (content)) == -1) +- exit (EXIT_FAILURE); +- +- if (guestfs_mkdir (g, "/fill") == -1) +- exit (EXIT_FAILURE); +- +- if (guestfs_fill_dir (g, "/fill", 10000) == -1) +- exit (EXIT_FAILURE); +- +- /* Force write through to and read back from disk. */ +- if (guestfs_sync (g) == -1 || guestfs_drop_caches (g, 3) == -1) +- exit (EXIT_FAILURE); +- +- data = guestfs_cat (g, filename); +- if (!data) +- exit (EXIT_FAILURE); +- +- if (strcmp (data, content) != 0) { +- fprintf (stderr, +- "%s FAILED: unexpected content of %s file " +- "(actual: %s, expected: %s)\n", +- program_name, filename, data, content); +- exit (EXIT_FAILURE); +- } +- +- files = guestfs_find (g, "/"); +- if (!files) +- exit (EXIT_FAILURE); +- for (i = 0; files[i] != NULL; ++i) +- free (files[i]); +- free (files); +- +-#ifdef GUESTFS_HAVE_FSTRIM +- /* Delete the files and fstrim to test zeroing/trimming. */ +- if (guestfs_rm (g, filename) == -1) +- exit (EXIT_FAILURE); +- +- if (guestfs_rm_rf (g, "/fill") == -1) +- exit (EXIT_FAILURE); +- +- if (guestfs_fstrim (g, "/", -1) == -1) +- exit (EXIT_FAILURE); +-#endif +- +- if (guestfs_shutdown (g) == -1) +- exit (EXIT_FAILURE); +- +- guestfs_close (g); +- exit (EXIT_SUCCESS); +-} +diff --git a/tests/test-readahead.sh b/tests/test-readahead.sh +new file mode 100755 +index 0000000..182aa00 +--- /dev/null ++++ b/tests/test-readahead.sh +@@ -0,0 +1,67 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2018-2019 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++# Is the readahead filter faster? Copy a blank disk with a custom ++# plugin that sleeps on every request. Because the readahead filter ++# should result in fewer requests it should run faster. ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires python3 --version ++requires python3 -c 'import nbd' ++ ++files="readahead.img" ++rm -f $files ++cleanup_fn rm -f $files ++ ++test () ++{ ++ start_t=$SECONDS ++ nbdkit -fv -U - "$@" sh ./test-readahead-test-plugin.sh \ ++ --run './test-readahead-test-request.py $unixsocket' ++ end_t=$SECONDS ++ echo $((end_t - start_t)) ++} ++ ++t1=$(test --filter=readahead) ++t2=$(test) ++ ++# In the t1 case we should make only 1 request into the plugin, ++# resulting in around 1 sleep period (5 seconds). In the t2 case we ++# make 10 requests so sleep for around 50 seconds. t1 should be < t2 ++# is every reasonable scenario. ++if [ $t1 -ge $t2 ]; then ++ echo "$0: readahead filter took longer, should be shorter" ++ exit 1 ++fi +-- +2.23.0 + diff --git a/nbdkit.spec b/nbdkit.spec index 4ab1563..544e704 100644 --- a/nbdkit.spec +++ b/nbdkit.spec @@ -31,14 +31,14 @@ ExclusiveArch: x86_64 %global verify_tarball_signature 1 # If there are patches which touch autotools files, set this to 1. -%global patches_touch_autotools %{nil} +%global patches_touch_autotools 1 # The source directory. %global source_directory 1.17-development Name: nbdkit Version: 1.17.4 -Release: 1%{?dist} +Release: 2%{?dist} Summary: NBD server License: BSD @@ -51,6 +51,10 @@ Source1: http://libguestfs.org/download/nbdkit/%{source_directory}/%{name Source2: libguestfs.keyring %endif +# Upstream patches to improve test times. +Patch1: 0001-tests-Reduce-excessive-time-taken-by-some-tests.patch +Patch2: 0002-tests-Improve-test-readahead-test.patch + %if 0%{patches_touch_autotools} BuildRequires: autoconf, automake, libtool %endif @@ -915,6 +919,9 @@ make %{?_smp_mflags} check || { %changelog +* Sat Dec 14 2019 Richard W.M. Jones - 1.17.4-2 +- Improve test times. + * Fri Dec 13 2019 Richard W.M. Jones - 1.17.4-1 - New upstream development version 1.17.4. - New filter: nofilter.