4f15d62cdd
The nightly scripts try to send an announcement about compose starting. As this happens before Pungi even started, the compose ID is not known and can not be included in the message. The message about starting rsync sent later will contain the ID correctly. Without the json defined there is a traceback in the cron output, but the errors are not logged anywhere else (but don't actually block anything in the compose process). Relates: https://pagure.io/pungi/issue/698 Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
168 lines
6.7 KiB
Bash
Executable File
168 lines
6.7 KiB
Bash
Executable File
#!/bin/sh
|
|
|
|
|
|
export LC_ALL=C
|
|
|
|
CONFIG="fedora.conf"
|
|
TARGET_DIR="/mnt/koji/compose/rawhide"
|
|
#OLD_COMPOSES_DIR="--old-composes=/mnt/fedora_koji/compose/f23 --old-composes=$TARGET_DIR"
|
|
NIGHTLY="--nightly"
|
|
SKIP_PHASES="--skip-phase=productimg"
|
|
DEST=$(pwd)
|
|
DATE=$(date "+%Y%m%d")
|
|
DIST="rawhide"
|
|
BRANCHED="rawhide"
|
|
BRANCH="rawhide"
|
|
COMPSFILE="comps-rawhide.xml"
|
|
GIT_BRANCH="master"
|
|
TMPDIR=`mktemp -d /tmp/$DIST.$DATE.XXXX`
|
|
TOMAIL="devel@lists.fedoraproject.org test@lists.fedoraproject.org"
|
|
FROM="Fedora Rawhide Report <rawhide@fedoraproject.org>"
|
|
RSYNCPREFIX="sudo -u ftpsync"
|
|
ATOMIC=$(mktemp -d /tmp/atomic.${DIST}.$DATE.XXXX)
|
|
ATOMICREPO="/mnt/koji/compose/atomic/$BRANCHED/"
|
|
ATOMICDEST="/mnt/koji/atomic/$BRANCHED/"
|
|
OLDCOMPOSE_ID=$(cat $TARGET_DIR/latest-Fedora-Rawhide/COMPOSE_ID)
|
|
# uncomment and edit for resuming a failed compose
|
|
#COMPOSE_ID="Fedora-23-20150530.n.0"
|
|
|
|
# assume a releng dir is a git checkout of the releng repo
|
|
# if it does not exist clone it
|
|
if [ -d releng ]; then
|
|
pushd releng
|
|
git pull --rebase
|
|
popd
|
|
else
|
|
git clone https://pagure.io/releng.git
|
|
fi
|
|
|
|
# Set up our fedmsg function, using the releng repo definition
|
|
FEDMSG_MODNAME="compose"
|
|
FEDMSG_CERTPREFIX="releng"
|
|
. ./releng/scripts/fedmsg-functions.sh
|
|
|
|
# Announce that we are starting, even though we don't know the compose_id yet..
|
|
fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s"}' "$BRANCHED" "$ARCH")
|
|
send_fedmsg "${fedmsg_json_start}" ${DIST} start
|
|
|
|
pushd $TMPDIR
|
|
git clone https://pagure.io/fedora-comps.git && {
|
|
pushd fedora-comps
|
|
make "${COMPSFILE}"
|
|
cp "${COMPSFILE}" $DEST/
|
|
popd
|
|
}
|
|
popd
|
|
|
|
./releng/scripts/block_retired.py
|
|
./releng/scripts/block_retired.py --namespace=docker
|
|
./releng/scripts/block_retired.py --namespace=container
|
|
|
|
CMD="pungi-koji --notification-script=/usr/bin/pungi-fedmsg-notification --config=$CONFIG --old-composes=$TARGET_DIR $OLD_COMPOSES_DIR $NIGHTLY $SKIP_PHASES"
|
|
|
|
if [ -z "$COMPOSE_ID" ]; then
|
|
CMD="$CMD --target-dir=$TARGET_DIR"
|
|
else
|
|
CMD="$CMD --debug-mode --compose-dir=$TARGET_DIR/$COMPOSE_ID"
|
|
fi
|
|
|
|
time $CMD "$@"
|
|
if [ "$?" != "0" ]; then
|
|
exit 1
|
|
fi
|
|
|
|
NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-Fedora-Rawhide/COMPOSE_ID)
|
|
SHORTCOMPOSE_ID=$(echo $NEWCOMPOSE_ID|sed -e 's|Fedora-.*-||g')
|
|
fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "compose_id": "%s"}' "$BRANCHED" "$ARCH" "$NEWCOMPOSE_ID")
|
|
fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "compose_id": "%s"}' "$BRANCHED" "$ARCH" "$NEWCOMPOSE_ID")
|
|
|
|
# Set this to use later for a few items include depcheck
|
|
DESTDIR=$TARGET_DIR/$NEWCOMPOSE_ID
|
|
|
|
compose-changelog -p $TARGET_DIR/$NEWCOMPOSE_ID/logs/ $TARGET_DIR/$OLDCOMPOSE_ID/ $TARGET_DIR/$NEWCOMPOSE_ID/
|
|
/usr/share/mash/spam-o-matic $DESTDIR/compose/Everything/ >$DESTDIR/logs/depcheck
|
|
|
|
[ -z "$ARCH" ] && {
|
|
./releng/scripts/critpath.py --url file://$DESTDIR/compose/Everything -o $DESTDIR/logs/critpath.txt branched &> $DESTDIR/logs/critpath.log
|
|
}
|
|
|
|
ostree prune --repo=$ATOMICREPO --keep-younger-than="14 days ago" --refs-only
|
|
|
|
# Tell interested persons that the rsync is starting (zomg!)
|
|
send_fedmsg "${fedmsg_json_start}" ${DIST} rsync.start
|
|
|
|
# Sync the content to /pub/fedora
|
|
if [ ! -d /pub/fedora/linux/development/$DIST ]; then
|
|
mkdir /pub/fedora/linux/development/$DIST
|
|
fi
|
|
$RSYNCPREFIX compose-partial-copy --arch=armhfp --arch=x86_64 --arch src \
|
|
"$TARGET_DIR/$NEWCOMPOSE_ID" /pub/fedora/linux/development/$DIST/ \
|
|
--variant Everything --variant Atomic --variant CloudImages --variant Docker --variant Server --variant Spins --variant Workstation \
|
|
--link-dest=/pub/fedora/linux/development/$DIST/Everything --exclude=repodata
|
|
$RSYNCPREFIX compose-partial-copy --arch=armhfp --arch=x86_64 --arch src \
|
|
"$TARGET_DIR/$NEWCOMPOSE_ID" /pub/fedora/linux/development/$DIST/ \
|
|
--variant Everything --variant Atomic --variant CloudImages --variant Docker --variant Server --variant Spins --variant Workstation \
|
|
--link-dest=/pub/fedora/linux/development/$DIST/Everything --delete-after
|
|
$RSYNCPREFIX rm /pub/fedora/linux/development/$DIST/.composeinfo
|
|
$RSYNCPREFIX ./releng/scripts/build_composeinfo /pub/fedora/linux/development/$DIST/ --name "$NEWCOMPOSE_ID"
|
|
|
|
# Sync the content to /pub/alt
|
|
if [ ! -d /pub/alt/development/$DIST ]; then
|
|
mkdir /pub/alt/development/$DIST
|
|
fi
|
|
$RSYNCPREFIX compose-partial-copy --arch=armhfp --arch=x86_64 \
|
|
"$TARGET_DIR/$NEWCOMPOSE_ID" /pub/alt/development/$DIST/ \
|
|
--variant Cloud --variant Labs \
|
|
--link-dest=/pub/fedora/linux/development/$DIST/Everything/ --exclude=repodata
|
|
$RSYNCPREFIX compose-partial-copy --arch=armhfp --arch=x86_64 \
|
|
"$TARGET_DIR/$NEWCOMPOSE_ID" /pub/alt/development/$DIST/ \
|
|
--variant Cloud --variant Labs \
|
|
--link-dest=/pub/fedora/linux/development/$DIST/Everything/ --delete-after
|
|
$RSYNCPREFIX rm /pub/alt/development/$DIST/.composeinfo
|
|
$RSYNCPREFIX ./releng/scripts/build_composeinfo /pub/alt/development/$DIST/ --name "$NEWCOMPOSE_ID"
|
|
|
|
# Sync the content to /pub/fedora-secondary
|
|
if [ ! -d /pub/fedora-secondary/development/$DIST ]; then
|
|
mkdir /pub/fedora-secondary/development/$DIST
|
|
fi
|
|
$RSYNCPREFIX compose-partial-copy --arch=aarch64 --arch=i386 --arch=ppc64 --arch=ppc64le \
|
|
"$TARGET_DIR/$NEWCOMPOSE_ID" /pub/fedora-secondary/development/$DIST/ \
|
|
--exclude=s390x \
|
|
--link-dest=/pub/fedora/linux/development/$DIST/Everything/ --exclude=repodata
|
|
$RSYNCPREFIX compose-partial-copy --arch=aarch64 --arch=i386 --arch=ppc64 --arch=ppc64le \
|
|
"$TARGET_DIR/$NEWCOMPOSE_ID" /pub/fedora-secondary/development/$DIST/ \
|
|
--exclude=s390x \
|
|
--link-dest=/pub/fedora/linux/development/$DIST/Everything/ --delete-after
|
|
$RSYNCPREFIX rm /pub/fedora-secondary/development/$DIST/.composeinfo
|
|
$RSYNCPREFIX ./releng/scripts/build_composeinfo /pub/fedora-secondary/development/$DIST/ --name "$NEWCOMPOSE_ID"
|
|
|
|
$RSYNCPREFIX rsync -avhH --delete-after $ATOMICREPO $ATOMICDEST
|
|
|
|
# Tell interested persons that the rsync is done.
|
|
send_fedmsg "${fedmsg_json_done}" ${DIST} rsync.complete
|
|
|
|
# Tell everyone by fedmsg about the compose
|
|
send_fedmsg "${fedmsg_json_done}" ${DIST} complete
|
|
|
|
# Tell everyone by email about the compose
|
|
SUBJECT='Fedora '$DIST' compose report: '$SHORTCOMPOSE_ID' changes'
|
|
for tomail in $TOMAIL ; do
|
|
cat $TARGET_DIR/$NEWCOMPOSE_ID/logs/*verbose $DESTDIR/logs/depcheck | \
|
|
mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail
|
|
done
|
|
|
|
for koji in arm ppc s390
|
|
do
|
|
if [ "$koji" = "arm" ]; then
|
|
arches=aarch64
|
|
elif [ "$koji" = "ppc" ]; then
|
|
arches=ppc64,ppc64le
|
|
elif [ "$koji" = "s390" ]; then
|
|
arches=s390x
|
|
fi
|
|
./releng/scripts/srpm-excluded-arch.py -a $arches --path $DESTDIR/compose/Everything/source/tree/Packages/ >$DESTDIR/logs/excludearch-$koji.log
|
|
done
|
|
|
|
find $TARGET_DIR -xdev -depth -maxdepth 2 -mtime +14 -exec rm -rf {} \;
|
|
|