From 356df95bb6c29dab529d98339aa495c38a5b95b1 Mon Sep 17 00:00:00 2001 From: Jan Kurik Date: Mon, 16 Jan 2023 18:48:52 +0100 Subject: [PATCH] Reworked gating plan to support upgrade testing - Allowed multiple plans - CI Gating tied up with gating.fmf plan - Added basic framework for upgrade testing - Testcases tagged to make those linked with plans --- ci.fmf | 1 + gating.yaml | 14 +--- plans/{ci.fmf => gating.fmf} | 1 + .../Sanity/SASL-authentication-check/main.fmf | 4 +- tests/Sanity/pcp-python-compliance/main.fmf | 4 +- tests/Sanity/pcp-services/main.fmf | 4 +- tests/Sanity/pcp-testsuite-sanity/main.fmf | 2 + tests/Sanity/rpm-config-files/main.fmf | 6 +- .../main.fmf | 4 +- tests/Sanity/upgrade/PURPOSE | 3 + tests/Sanity/upgrade/main.fmf | 33 +++++++++ tests/Sanity/upgrade/runtest.sh | 73 +++++++++++++++++++ tests/Sanity/verify-systemd-units/main.fmf | 4 +- tests/Sanity/writeable-suid-guid/main.fmf | 4 +- 14 files changed, 129 insertions(+), 28 deletions(-) create mode 100644 ci.fmf rename plans/{ci.fmf => gating.fmf} (75%) create mode 100644 tests/Sanity/upgrade/PURPOSE create mode 100644 tests/Sanity/upgrade/main.fmf create mode 100755 tests/Sanity/upgrade/runtest.sh diff --git a/ci.fmf b/ci.fmf new file mode 100644 index 0000000..c5aa0e0 --- /dev/null +++ b/ci.fmf @@ -0,0 +1 @@ +resultsdb-testcase: separate diff --git a/gating.yaml b/gating.yaml index ce3cdc1..116ac6c 100644 --- a/gating.yaml +++ b/gating.yaml @@ -4,16 +4,4 @@ product_versions: decision_context: bodhi_update_push_stable subject_type: koji_build rules: - - !PassingTestCaseRule {test_case_name: fedora-ci.koji-build.tier0.functional} ---- !Policy -product_versions: - - rhel-8 -decision_context: osci_compose_gate -rules: - - !PassingTestCaseRule {test_case_name: baseos-ci.brew-build.tier1.functional} ---- !Policy -product_versions: - - rhel-9 -decision_context: osci_compose_gate -rules: - - !PassingTestCaseRule {test_case_name: baseos-ci.brew-build.tier1.functional} + - !PassingTestCaseRule {test_case_name: fedora-ci.koji-build./plans/gating.functional} diff --git a/plans/ci.fmf b/plans/gating.fmf similarity index 75% rename from plans/ci.fmf rename to plans/gating.fmf index 3fd3ab7..5ed0c31 100644 --- a/plans/ci.fmf +++ b/plans/gating.fmf @@ -1,5 +1,6 @@ summary: CI Gating Plan discover: how: fmf + filter: tag:gating execute: how: tmt diff --git a/tests/Sanity/SASL-authentication-check/main.fmf b/tests/Sanity/SASL-authentication-check/main.fmf index e7fc118..0b87aed 100644 --- a/tests/Sanity/SASL-authentication-check/main.fmf +++ b/tests/Sanity/SASL-authentication-check/main.fmf @@ -11,5 +11,5 @@ recommend: - cyrus-sasl-scram - cyrus-sasl-lib duration: 15m -extra-summary: /tools/pcp/Sanity/SASL-authentication-check -extra-task: /tools/pcp/Sanity/SASL-authentication-check +tag: + - gating diff --git a/tests/Sanity/pcp-python-compliance/main.fmf b/tests/Sanity/pcp-python-compliance/main.fmf index 87bd429..d9f5e19 100644 --- a/tests/Sanity/pcp-python-compliance/main.fmf +++ b/tests/Sanity/pcp-python-compliance/main.fmf @@ -9,5 +9,5 @@ recommend: - python3-pcp - python-pcp duration: 15m -extra-summary: /tools/pcp/Sanity/pcp-python-compliance -extra-task: /tools/pcp/Sanity/pcp-python-compliance +tag: + - gating diff --git a/tests/Sanity/pcp-services/main.fmf b/tests/Sanity/pcp-services/main.fmf index 1fe246c..28ced58 100644 --- a/tests/Sanity/pcp-services/main.fmf +++ b/tests/Sanity/pcp-services/main.fmf @@ -12,5 +12,5 @@ recommend: - pcp-zeroconf - redis duration: 15m -extra-summary: /tools/pcp/Sanity/pcp-services -extra-task: /tools/pcp/Sanity/pcp-services +tag: + - gating diff --git a/tests/Sanity/pcp-testsuite-sanity/main.fmf b/tests/Sanity/pcp-testsuite-sanity/main.fmf index 62618b6..1109ebb 100644 --- a/tests/Sanity/pcp-testsuite-sanity/main.fmf +++ b/tests/Sanity/pcp-testsuite-sanity/main.fmf @@ -27,3 +27,5 @@ recommend: - /usr/bin/netstat - pcp-testsuite duration: 4h +tag: + - gating diff --git a/tests/Sanity/rpm-config-files/main.fmf b/tests/Sanity/rpm-config-files/main.fmf index 2e12b7f..7356d43 100644 --- a/tests/Sanity/rpm-config-files/main.fmf +++ b/tests/Sanity/rpm-config-files/main.fmf @@ -1,4 +1,4 @@ -summary: Verification of rpm's config files +summary: Verification of rpms config files description: 'Verification of rpms config files' contact: Jan Kuřík component: @@ -9,5 +9,5 @@ recommend: - pcp - rpm-build duration: 15m -extra-summary: /tools/pcp/Sanity/rpm-config-files -extra-task: /tools/pcp/Sanity/rpm-config-files +tag: + - gating diff --git a/tests/Sanity/shared-libraries-need-symbol-versioning/main.fmf b/tests/Sanity/shared-libraries-need-symbol-versioning/main.fmf index cb2ea2b..03a1c0e 100644 --- a/tests/Sanity/shared-libraries-need-symbol-versioning/main.fmf +++ b/tests/Sanity/shared-libraries-need-symbol-versioning/main.fmf @@ -12,5 +12,5 @@ recommend: - pcp-libs - binutils duration: 15m -extra-summary: /tools/pcp/Sanity/shared-libraries-need-symbol-versioning -extra-task: /tools/pcp/Sanity/shared-libraries-need-symbol-versioning +tag: + - gating diff --git a/tests/Sanity/upgrade/PURPOSE b/tests/Sanity/upgrade/PURPOSE new file mode 100644 index 0000000..dc63252 --- /dev/null +++ b/tests/Sanity/upgrade/PURPOSE @@ -0,0 +1,3 @@ +PURPOSE of /Sanity/upgrade +Description: upgrade test between different distro releases +Author: Jan Kurik diff --git a/tests/Sanity/upgrade/main.fmf b/tests/Sanity/upgrade/main.fmf new file mode 100644 index 0000000..182740a --- /dev/null +++ b/tests/Sanity/upgrade/main.fmf @@ -0,0 +1,33 @@ +summary: upgrade +description: '' +contact: Jan Kurik +component: + - pcp +test: ./runtest.sh +framework: beakerlib +require: + - library(ControlFlow/mcase) +recommend: + - pcp + - beakerlib + - time + - sudo + - procps-ng + - avahi + - bzip2 + - pcp-pmda-cisco + - coreutils + - pcp-pmda-news + - perl-PCP-LogImport + - perl-PCP-LogSummary + - python-pcp + - gcc + - redhat-rpm-config + - perl-PCP-LogSummary + - libuv + - libuv-devel + - /usr/bin/netstat + - pcp-testsuite +duration: 1h +tag: + - upgrade diff --git a/tests/Sanity/upgrade/runtest.sh b/tests/Sanity/upgrade/runtest.sh new file mode 100755 index 0000000..164263d --- /dev/null +++ b/tests/Sanity/upgrade/runtest.sh @@ -0,0 +1,73 @@ +#!/bin/bash +#shellcheck disable=SC1091 +# vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# runtest.sh of /CoreOS/pcp/Sanity/upgrade +# Description: upgrade +# Author: Jan Kurik +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Copyright (c) 2022 Red Hat, Inc. +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation, either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +# Include Beaker environment +. /usr/share/beakerlib/beakerlib.sh || exit 1 +. ../../Library/pcpcommon/lib.sh || exit 1 + +#shellcheck disable=SC2034 +PACKAGE=pcp + +distribution_mcase__test() { + rlLogInfo 'Verify scenario upgrade works' + + rlRun "pcpcommonLibraryLoaded" + rlRun "rlServiceStart pmcd pmlogger" 0,1 + rlRun "rlServiceEnable pmcd pmlogger" 0-255 + rlRun "sleep 10" + + rlRun "pcpcommon_test -g sanity -g pmda.linux" || rlRun "pcpcommon_log_system_info" + rlRun "pcpcommon_cleanup" +} + +rlJournalStart + rlPhaseStartSetup "init" + + export LANG=en_US.UTF-8 + export LANGUAGE=en_US:en + export LC_CTYPE=POSIX + export LC_NUMERIC=POSIX + export LC_TIME=POSIX + export LC_COLLATE=POSIX + export LC_MONETARY=POSIX + export LC_MESSAGES=POSIX + export LC_PAPER=POSIX + export LC_NAME=POSIX + export LC_ADDRESS=POSIX + export LC_TELEPHONE=POSIX + export LC_MEASUREMENT=POSIX + export LC_IDENTIFICATION=POSIX + export LC_ALL= + + rlImport "ControlFlow/mcase" + rlPhaseEnd + distribution_mcase__run +rlJournalPrintText +rlJournalEnd + +#template by morf-0.29.25 diff --git a/tests/Sanity/verify-systemd-units/main.fmf b/tests/Sanity/verify-systemd-units/main.fmf index 1d9628d..d496a66 100644 --- a/tests/Sanity/verify-systemd-units/main.fmf +++ b/tests/Sanity/verify-systemd-units/main.fmf @@ -11,5 +11,5 @@ recommend: - pcp - pcp-zeroconf duration: 5m -extra-summary: /tools/pcp/Sanity/verify-systemd-units -extra-task: /tools/pcp/Sanity/verify-systemd-units +tag: + - gating diff --git a/tests/Sanity/writeable-suid-guid/main.fmf b/tests/Sanity/writeable-suid-guid/main.fmf index 69bae82..54c72eb 100644 --- a/tests/Sanity/writeable-suid-guid/main.fmf +++ b/tests/Sanity/writeable-suid-guid/main.fmf @@ -11,5 +11,5 @@ recommend: - pcp - pcp-testsuite duration: 15m -extra-summary: /tools/pcp/Sanity/writeable-suid-guid -extra-task: /tools/pcp/Sanity/writeable-suid-guid +tag: + - gating