pungi-fedora/nightly.sh

#!/bin/sh


export LC_ALL=C

CONFIG="fedora.conf"
TARGET_DIR="/mnt/koji/compose/rawhide"
#OLD_COMPOSES_DIR="--old-composes=/mnt/fedora_koji/compose/f23 --old-composes=$TARGET_DIR"
NIGHTLY="--nightly"
SKIP_PHASES="--skip-phase=productimg"
DEST=$(pwd)
DATE=$(date "+%Y%m%d")
SHORT="Fedora"
RELEASE="rawhide"
RELEASE_TITLE="Rawhide"
COMPSFILE="comps-rawhide.xml"
TMPDIR=`mktemp -d /tmp/$RELEASE.$DATE.XXXX`
TOMAIL="devel@lists.fedoraproject.org test@lists.fedoraproject.org"
FROM="Fedora Rawhide Report <rawhide@fedoraproject.org>"
RSYNCPREFIX="sudo -u ftpsync"
ATOMIC=$(mktemp -d /tmp/atomic.${RELEASE}.$DATE.XXXX)
ATOMICHOSTREPO="/mnt/koji/compose/atomic/$RELEASE/"
ATOMICWSREPO="/mnt/koji/compose/ostree/$RELEASE/"
ATOMICHOSTDEST="/mnt/koji/atomic/$RELEASE/"
OLDCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)
# uncomment and edit for resuming a failed compose
#COMPOSE_ID="Fedora-23-20150530.n.0"

# assume a releng dir is a git checkout of the releng repo
# if it does not exist clone it
if [ -d releng ]; then
    pushd releng
    git pull --rebase
    popd
else
    git clone https://pagure.io/releng.git
fi

# Set up our fedmsg function, using the releng repo definition
FEDMSG_MODNAME="compose"
FEDMSG_CERTPREFIX="releng"
. ./releng/scripts/fedmsg-functions.sh

# Announce that we are starting, even though we don't know the compose_id yet..
fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s"}' "$RELEASE" "$ARCH" "$SHORT")
send_fedmsg "${fedmsg_json_start}" ${RELEASE} start

#pushd $TMPDIR
#git clone https://pagure.io/fedora-comps.git && {
#    pushd fedora-comps
#    make "${COMPSFILE}"
#    cp "${COMPSFILE}" $DEST/
#    popd
#}
#popd

./releng/scripts/block_retired.py --profile compose_koji
./releng/scripts/block_retired.py --profile compose_koji --namespace=docker
./releng/scripts/block_retired.py --profile compose_koji --namespace=container

CMD="pungi-koji --notification-script=/usr/bin/pungi-fedmsg-notification --notification-script=pungi-wait-for-signed-ostree-handler --config=$CONFIG --old-composes=$TARGET_DIR $OLD_COMPOSES_DIR $NIGHTLY $SKIP_PHASES"

if [ -z "$COMPOSE_ID" ]; then
    CMD="$CMD --target-dir=$TARGET_DIR"
else
    CMD="$CMD --debug-mode --compose-dir=$TARGET_DIR/$COMPOSE_ID"
fi

time $CMD "$@"
if [ "$?" != "0" ]; then
    exit 1
fi

NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)
SHORTCOMPOSE_ID=$(echo $NEWCOMPOSE_ID|sed -e 's|Fedora-.*-||g')
fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")
fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

# Set this to use later for a few items include depcheck
DESTDIR=$TARGET_DIR/$NEWCOMPOSE_ID

compose-changelog -p $TARGET_DIR/$NEWCOMPOSE_ID/logs/ $TARGET_DIR/$OLDCOMPOSE_ID/ $TARGET_DIR/$NEWCOMPOSE_ID/
# Figure out a version for broken deps e-mail that goes to package maintainers.
# In Rawhide it's just rawhide, for branched versions we prepend F- to the number.
if [ "$RELEASE" = "rawhide" ]; then
    TREENAME="$RELEASE"
else
    TREENAME="F-$RELEASE"
fi
/usr/share/mash/spam-o-matic --treename="$TREENAME" "$DESTDIR/compose/Everything/" >"$DESTDIR/logs/depcheck"

[ -z "$ARCH" ] && {
./releng/scripts/critpath.py --url file://$DESTDIR/compose/Everything/ -o $DESTDIR/logs/critpath.txt rawhide &> $DESTDIR/logs/critpath.log
}

# prune atomic host and atomic workstation repos
ostree prune --repo=$ATOMICHOSTREPO --keep-younger-than="14 days ago" --refs-only
ostree prune --repo=$ATOMICWSREPO --keep-younger-than="14 days ago" --refs-only

# Tell interested persons that the rsync is starting (zomg!)
send_fedmsg "${fedmsg_json_start}" ${RELEASE} rsync.start

# Sync the content to /pub/fedora
if [ ! -d /pub/fedora/linux/development/$RELEASE ]; then
  mkdir /pub/fedora/linux/development/$RELEASE
fi
$RSYNCPREFIX compose-partial-copy --arch=armhfp --arch=x86_64 --arch src \
    "$TARGET_DIR/$NEWCOMPOSE_ID" /pub/fedora/linux/development/$RELEASE/ \
    --variant Everything --variant Atomic --variant CloudImages --variant Docker --variant Server --variant Spins --variant Workstation \
    --link-dest=/pub/fedora/linux/development/$RELEASE/Everything --exclude=repodata
$RSYNCPREFIX compose-partial-copy --arch=armhfp --arch=x86_64 --arch src \
    "$TARGET_DIR/$NEWCOMPOSE_ID" /pub/fedora/linux/development/$RELEASE/ \
    --variant Everything --variant Atomic --variant CloudImages --variant Docker --variant Server --variant Spins --variant Workstation \
    --link-dest=/pub/fedora/linux/development/$RELEASE/Everything --delete-after
$RSYNCPREFIX rm /pub/fedora/linux/development/$RELEASE/.composeinfo
$RSYNCPREFIX ./releng/scripts/build_composeinfo /pub/fedora/linux/development/$RELEASE/ --name "$NEWCOMPOSE_ID"

# Sync the content to /pub/alt
if [ ! -d /pub/alt/development/$RELEASE ]; then
  mkdir /pub/alt/development/$RELEASE
fi
$RSYNCPREFIX compose-partial-copy --arch=armhfp --arch=x86_64 \
    "$TARGET_DIR/$NEWCOMPOSE_ID" /pub/alt/development/$RELEASE/ \
    --variant Cloud --variant Labs \
    --link-dest=/pub/fedora/linux/development/$RELEASE/Everything/ --exclude=repodata
$RSYNCPREFIX compose-partial-copy --arch=armhfp --arch=x86_64 \
    "$TARGET_DIR/$NEWCOMPOSE_ID" /pub/alt/development/$RELEASE/ \
    --variant Cloud --variant Labs \
    --link-dest=/pub/fedora/linux/development/$RELEASE/Everything/ --delete-after
$RSYNCPREFIX rm /pub/alt/development/$RELEASE/.composeinfo
$RSYNCPREFIX ./releng/scripts/build_composeinfo /pub/alt/development/$RELEASE/ --name "$NEWCOMPOSE_ID"

# Sync the content to /pub/fedora-secondary
if [ ! -d /pub/fedora-secondary/development/$RELEASE ]; then
  mkdir /pub/fedora-secondary/development/$RELEASE
fi
$RSYNCPREFIX compose-partial-copy --arch=aarch64 --arch=i386 --arch=ppc64 --arch=ppc64le --arch=s390x \
    "$TARGET_DIR/$NEWCOMPOSE_ID" /pub/fedora-secondary/development/$RELEASE/ \
    --link-dest=/pub/fedora/linux/development/$RELEASE/Everything/ --exclude=repodata
$RSYNCPREFIX compose-partial-copy --arch=aarch64 --arch=i386 --arch=ppc64 --arch=ppc64le --arch=s390x \
    "$TARGET_DIR/$NEWCOMPOSE_ID" /pub/fedora-secondary/development/$RELEASE/ \
    --link-dest=/pub/fedora/linux/development/$RELEASE/Everything/ --delete-after
$RSYNCPREFIX rm /pub/fedora-secondary/development/$RELEASE/.composeinfo
$RSYNCPREFIX ./releng/scripts/build_composeinfo /pub/fedora-secondary/development/$RELEASE/ --name "$NEWCOMPOSE_ID"

$RSYNCPREFIX rsync -avhH --delete-after $ATOMICHOSTREPO $ATOMICHOSTDEST

# Tell interested persons that the rsync is done.
send_fedmsg "${fedmsg_json_done}" ${RELEASE} rsync.complete

# Tell everyone by fedmsg about the compose
send_fedmsg "${fedmsg_json_done}" ${RELEASE} complete

# Tell everyone by email about the compose
SUBJECT='Fedora '$RELEASE' compose report: '$SHORTCOMPOSE_ID' changes'
for tomail in $TOMAIL ; do
    cat $TARGET_DIR/$NEWCOMPOSE_ID/logs/*verbose $DESTDIR/logs/depcheck | \
         mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail
done

for koji in arm ppc s390
do
  if [ "$koji" = "arm" ]; then
     arches=aarch64
  elif [ "$koji" = "ppc" ]; then
     arches=ppc64,ppc64le
  elif [ "$koji" = "s390" ]; then
     arches=s390x
  fi
  ./releng/scripts/srpm-excluded-arch.py -a $arches --path $DESTDIR/compose/Everything/source/tree/Packages/ >$DESTDIR/logs/excludearch-$koji.log
done

find $TARGET_DIR  -xdev -depth -maxdepth 2 -mtime +14  -exec rm -rf {} \;