make delta update work

This commit is contained in:
Harald Hoyer 2018-11-15 16:47:59 +01:00
parent a911d96bd2
commit b1d9041579
4 changed files with 237 additions and 146 deletions

View file

@ -60,32 +60,38 @@ PROGNAME=${0##*/}
BASEDIR=$(realpath ${0%/*})
JSON="$(realpath -e $1)"
JSONDIR="${JSON%/*}"
DISTDIR="${JSON%/*}"
NAME="$(jq -r '.name' ${JSON})"
VERSION="$(jq -r '.version' ${JSON})"
ROOTHASH="$(jq -r '.roothash' ${JSON})"
IMAGE="${JSONDIR}/${NAME}-${VERSION}"
IMAGE="${DISTDIR}/${NAME}-${VERSION}.json"
CRT=${CRT:-${BASEDIR}/${NAME}.crt}
KEY=${KEY:-${BASEDIR}/${NAME}.key}
mkdelta_f() {
OLD="$1"
NEW="$2"
if [[ -e "$OLD"/root-hash.txt ]]; then
DELTANAME="$JSONDIR/$NAME-$(<"$OLD"/root-hash.txt)"
else
DELTANAME="$JSONDIR/$NAME-"$(jq -r '.roothash' "$OLD"/release.json)""
fi
xdelta3 -9 -f -S djw -s "$OLD"/root.img "$NEW"/root.img "$DELTANAME"-delta.new
local OLD="$1"
local NEW="$2"
local DELTANAME="$DISTDIR/$NAME-$(jq -r '.roothash' "$OLD")"
local OLDIMAGE="$DISTDIR/$NAME-$(jq -r '.roothash' "$OLD").img"
local NEWHASH=$(jq -r '.roothash' "$NEW")
local NEWIMAGE="$DISTDIR/$NAME-$NEWHASH.img"
xdelta3 -9 -f -S djw -s "$OLDIMAGE" "$NEWIMAGE" "$DELTANAME"-delta.new
openssl dgst -sha256 -sign "$KEY" -out "$DELTANAME"-delta.new.sig "$DELTANAME"-delta.new
mv "$DELTANAME"-delta.new "$DELTANAME"-delta.img
mv "$DELTANAME"-delta.new.sig "$DELTANAME"-delta.img.sig
cp "${NEW}/release.json" "${DELTANAME}.json"
openssl dgst -sha256 -sign "$KEY" -out "${DELTANAME}.json.sig" "${DELTANAME}.json"
DELTA_IMAGE_SIZE=$(stat --printf '%s' "$DELTANAME"-delta.img)
jq "( . + {\
\"deltasig\": \"$(xxd -c256 -p -g0 < "$DELTANAME"-delta.new.sig)\",\
\"deltasize\": \"${DELTA_IMAGE_SIZE}\",\
})" \
< "${NEW}" > "${DELTANAME}-delta.json"
rm -f "$DELTANAME"-delta.new.sig
openssl dgst -sha256 -sign "$KEY" -out "${DELTANAME}-delta.json.sig" "${DELTANAME}-delta.json"
}
for i in $(ls -1d "${JSONDIR}/${NAME}-"*); do
[[ -d "$i" ]] || continue
for i in $(ls -1 "${DISTDIR}/${NAME}-"*.??????????????.json); do
[[ -f "$i" ]] || continue
OLDIMAGE=$(realpath $i)
if [[ $OLDIMAGE == $IMAGE ]]; then
@ -93,5 +99,14 @@ for i in $(ls -1d "${JSONDIR}/${NAME}-"*); do
fi
mkdelta_f "$OLDIMAGE" "$IMAGE"
[[ $CHECKPOINT ]] && rm -fr "$OLDIMAGE" "$OLDIMAGE".tgz "$OLDIMAGE"-efi.tgz "$OLDIMAGE"-efi.tgz.sig
if [[ $CHECKPOINT ]]; then
OLDHASH="$(jq -r '.roothash' "$OLDIMAGE")"
OLDNAME="$(jq -r '.name' "$OLDIMAGE")"
rm -f \
"$OLDIMAGE" \
"$OLDIMAGE".sig \
"${DISTDIR}/$OLDNAME"-"$OLDHASH".img \
"${DISTDIR}/$OLDNAME"-"$OLDHASH"-efi.tgz "${DISTDIR}/$OLDNAME"-"$OLDHASH"-efi.tgz.sig \
"${DISTDIR}/$OLDNAME"-"$OLDHASH".json "${DISTDIR}/$OLDNAME"-"$OLDHASH".json.sig
fi
done

View file

@ -15,8 +15,6 @@ TEMP=$(
getopt -o '' \
--long key: \
--long crt: \
--long nosign \
--long notar \
--long help \
-- "$@"
)
@ -39,14 +37,6 @@ while true; do
CRT="$(readlink -e $2)"
shift 2; continue
;;
'--nosign')
NOSIGN="1"
shift 1; continue
;;
'--notar')
NOTAR="1"
shift 1; continue
;;
'--help')
usage
exit 0
@ -65,43 +55,69 @@ PROGNAME=${0##*/}
BASEDIR=$(realpath ${0%/*})
JSON="$(realpath -e $1)"
JSONDIR="${JSON%/*}"
BASEOUTDIR="${JSON%/*}"
NAME="$(jq -r '.name' ${JSON})"
VERSION="$(jq -r '.version' ${JSON})"
ROOTHASH="$(jq -r '.roothash' ${JSON})"
IMAGE="${JSONDIR}/${NAME}-${VERSION}"
HASH_IMAGE="${JSONDIR}/${NAME}-${ROOTHASH}"
IMAGE="${BASEOUTDIR}/${NAME}-${VERSION}"
HASH_IMAGE="${BASEOUTDIR}/${NAME}-${ROOTHASH}"
CRT=${CRT:-${BASEDIR}/${NAME}.crt}
KEY=${KEY:-${BASEDIR}/${NAME}.key}
pushd "$IMAGE"
if ! [[ $NOSIGN ]]; then
[[ $TMPDIR ]] || TMPDIR=/var/tmp
readonly TMPDIR="$(realpath -e "$TMPDIR")"
[ -d "$TMPDIR" ] || {
printf "%s\n" "${PROGNAME}: Invalid tmpdir '$tmpdir'." >&2
exit 1
}
readonly MY_TMPDIR="$(mktemp -p "$TMPDIR/" -d -t ${PROGNAME}.XXXXXX)"
[ -d "$MY_TMPDIR" ] || {
printf "%s\n" "${PROGNAME}: mktemp -p '$TMPDIR/' -d -t ${PROGNAME}.XXXXXX failed." >&2
exit 1
}
# clean up after ourselves no matter how we die.
trap '
ret=$?;
[[ $MY_TMPDIR ]] && rm -rf --one-file-system -- "$MY_TMPDIR"
exit $ret;
' EXIT
# clean up after ourselves no matter how we die.
trap 'exit 1;' SIGINT
cd "$MY_TMPDIR"
if ! [[ $KEY ]] || ! [[ $CRT ]]; then
echo "Cannot find $KEY and $CRT"
echo "Need --key KEY --crt CRT options"
exit 1
fi
for i in $(find . -type f -name '*.efi'); do
tar xzf "${HASH_IMAGE}-efi.tgz"
for i in $(find efi -type f -name '*.efi'); do
[[ -f "$i" ]] || continue
if ! sbverify --cert "$CRT" "$i" &>/dev/null ; then
sbsign --key "$KEY" --cert "$CRT" --output "${i}signed" "$i"
mv "${i}signed" "$i"
fi
done
fi
[[ -f sha512sum.txt ]] || sha512sum $(find . -type f) > sha512sum.txt
[[ -f sha512sum.txt.sig ]] || openssl dgst -sha256 -sign "$KEY" -out sha512sum.txt.sig sha512sum.txt
rm "${HASH_IMAGE}-efi.tgz"
tar cf - efi | pigz -c > "${HASH_IMAGE}-efi.tgz"
if ! [[ $NOTAR ]]; then
[[ -e "$IMAGE".tgz ]] || tar cf - -C "${IMAGE%/*}" "${IMAGE##*/}" | pigz -c > "${IMAGE}.tgz"
if ! [[ -e "$HASH_IMAGE-efi".tgz ]]; then
tar cf - efi | pigz -c > "$HASH_IMAGE-efi.tgz"
fi
[[ $NOSIGN ]] || openssl dgst -sha256 -sign "$KEY" \
-out "${HASH_IMAGE}-efi.tgz.sig" "${HASH_IMAGE}-efi.tgz"
[[ $NOSIGN ]] || openssl dgst -sha256 -sign "$KEY" \
-out "${JSONDIR}/${NAME}-${ROOTHASH}.img.sig" "$IMAGE/root.img"
fi
openssl dgst -sha256 -sign "$KEY" \
-out efi.sig "${HASH_IMAGE}-efi.tgz"
popd
openssl dgst -sha256 -sign "$KEY" \
-out img.sig "${HASH_IMAGE}.img"
jq "( . + {\"efitarsig\": \"$(xxd -c256 -p -g0 \
< efi.sig)\"} + {\"rootimgsig\":\"$(xxd -c256 -p -g0 \
< img.sig)\"})" \
> "${IMAGE}.json.new" < "${IMAGE}.json" \
&& mv --force "${IMAGE}.json.new" "${IMAGE}.json"
openssl dgst -sha256 -sign "$KEY" \
-out "${IMAGE}.json.sig" "${IMAGE}.json"

View file

@ -10,7 +10,7 @@ Creates a directory with a readonly root on squashfs, a dm_verity file and an EF
--pkglist FILE The packages to install read from FILE (default: pkglist.txt)
--excludelist FILE The packages to install read from FILE (default: excludelist.txt)
--releasever NUM Used Fedora release version NUM (default: $VERSION_ID)
--outdir DIR Creates DIR and puts all files in there (default: NAME-NUM-DATE)
--outname JSON Creates \$JSON.json symlinked to that release (default: NAME-NUM-DATE)
--baseoutdir DIR Parent directory of --outdir
--name NAME The NAME of the product (default: FedoraBook)
--logo FILE Uses the .bmp FILE to display as a splash screen (default: logo.bmp)
@ -35,7 +35,7 @@ TEMP=$(
--long help \
--long pkglist: \
--long excludelist: \
--long outdir: \
--long outname: \
--long baseoutdir: \
--long name: \
--long releasever: \
@ -79,8 +79,8 @@ while true; do
fi
shift 2; continue
;;
'--outdir')
OUTDIR="$2"
'--outname')
OUTNAME="$2"
shift 2; continue
;;
'--baseoutdir')
@ -173,7 +173,7 @@ trap '
[[ -d "$i" ]] && mountpoint -q "$i" && umount "$i"
done
[[ $MY_TMPDIR ]] && rm -rf --one-file-system -- "$MY_TMPDIR"
(( $ret != 0 )) && [[ "$OUTDIR" ]] && rm -rf --one-file-system -- "$OUTDIR"
(( $ret != 0 )) && [[ "$OUTNAME" ]] && rm -rf --one-file-system -- "$OUTNAME"
setenforce $OLD_SELINUX
exit $ret;
' EXIT
@ -858,8 +858,8 @@ else
fi
VERSION_ID="${RELEASEVER}.$(date -u +'%Y%m%d%H%M%S' --date @$SOURCE_DATE_EPOCH)"
OUTDIR=${OUTDIR:-"${NAME}-${VERSION_ID}"}
OUTDIR="${BASEOUTDIR}/${OUTDIR}"
OUTNAME=${OUTNAME:-"${NAME}-${VERSION_ID}"}
OUTNAME="${BASEOUTDIR}/${OUTNAME}"
if [[ -f "$sysroot"/etc/os-release ]]; then
sed -i -e "s#VERSION_ID=.*#VERSION_ID=$VERSION_ID#" "$sysroot"/etc/os-release
@ -904,32 +904,33 @@ if ! [[ $EFISTUB ]]; then
fi
fi
[[ -e "$OUTDIR" ]] && rm -fr "$OUTDIR"
mkdir -p "$OUTDIR"
mv "$MY_TMPDIR"/root.img \
"$sysroot"/usr/efi \
"$OUTDIR"/
mkdir -p "$OUTDIR"/efi/EFI/${NAME}
mkdir -p "$sysroot"/usr/efi/EFI/${NAME}
objcopy \
--add-section .release="$MY_TMPDIR"/release.txt --change-section-vma .release=0x20000 \
--add-section .cmdline="$MY_TMPDIR"/options.txt --change-section-vma .cmdline=0x30000 \
${LOGO:+--add-section .splash="$LOGO" --change-section-vma .splash=0x40000} \
--add-section .linux="$sysroot"/lib/modules/$KVER/vmlinuz --change-section-vma .linux=0x2000000 \
--add-section .initrd="$sysroot"/lib/modules/$KVER/initrd --change-section-vma .initrd=0x3000000 \
"${EFISTUB}" "$OUTDIR"/efi/EFI/${NAME}/bootx64-$ROOT_HASH.efi
"${EFISTUB}" "$sysroot"/usr/efi/EFI/${NAME}/bootx64-$ROOT_HASH.efi
cat > "${OUTDIR}/release.json" <<EOF
tar cf - -C "$sysroot"/usr efi | pigz -c > "${BASEOUTDIR}/${NAME}-${ROOT_HASH}-efi.tgz"
mv "$MY_TMPDIR"/root.img "${BASEOUTDIR}/${NAME}-${ROOT_HASH}.img"
cat > "${OUTNAME}.json" <<EOF
{
"roothash": "$ROOT_HASH",
"rootsize": "$ROOT_SIZE",
"roothash": "${ROOT_HASH}",
"imagesize": "${IMAGE_SIZE}",
"name" : "${NAME}",
"version" : "${VERSION_ID}"
}
EOF
chown -R "$USER" "$OUTDIR"
ln -sfnr "${OUTNAME}.json" "${BASEOUTDIR}/${NAME}-latest.json"
chown "${SUDO_USER:-$USER}" \
"${OUTNAME}.json" \
"${BASEOUTDIR}/${NAME}-${ROOT_HASH}.img" \
"${BASEOUTDIR}/${NAME}-${ROOT_HASH}-efi.tgz" \
"${BASEOUTDIR}/${NAME}-latest.json"
cp -a "${OUTDIR}/release.json" "${BASEOUTDIR}/${NAME}-latest.json"
setenforce $OLD_SELINUX

195
update.sh
View file

@ -9,18 +9,14 @@ Usage: $PROGNAME [OPTION]
-h, --help Display this help
--force Update, even if the signature checks fail
--dir DIR Update from DIR, instead of downloading
--nocheck Do not check the integrity of the update data
--nodownload Use the existing *.json file in the current directory
--json JSON Update from JSON, instead of downloading
EOF
}
TEMP=$(
getopt -o '' \
--long dir: \
--long json: \
--long force \
--long nocheck \
--long nodownload \
--long help \
-- "$@"
)
@ -35,22 +31,14 @@ unset TEMP
while true; do
case "$1" in
'--dir')
USE_DIR="$(readlink -e $2)"
'--json')
USE_JSON="$(readlink -e $2)"
shift 2; continue
;;
'--force')
FORCE="y"
shift 1; continue
;;
'--nocheck')
NO_CHECK="y"
shift 1; continue
;;
'--nodownload')
NO_DOWNLOAD="y"
shift 1; continue
;;
'--help')
usage
exit 0
@ -74,6 +62,10 @@ CURRENT_ROOT_HASH=$(</proc/cmdline)
CURRENT_ROOT_HASH=${CURRENT_ROOT_HASH#*roothash=}
CURRENT_ROOT_HASH=${CURRENT_ROOT_HASH%% *}
CURRENT_IMAGE_SIZE=$(</proc/cmdline)
CURRENT_IMAGE_SIZE=${CURRENT_IMAGE_SIZE#*verity.imagesize=}
CURRENT_IMAGE_SIZE=${CURRENT_IMAGE_SIZE%% *}
CURRENT_ROOT_UUID=${CURRENT_ROOT_HASH:32:8}-${CURRENT_ROOT_HASH:40:4}-${CURRENT_ROOT_HASH:44:4}-${CURRENT_ROOT_HASH:48:4}-${CURRENT_ROOT_HASH:52:12}
bootdisk() {
@ -155,76 +147,143 @@ trap '
# clean up after ourselves no matter how we die.
trap 'exit 1;' SIGINT
if [[ $USE_DIR ]]; then
IMAGE="$USE_DIR"
ROOT_HASH=$(jq -r '.roothash' "$IMAGE"/release.json)
if ! [[ $FORCE ]] && [[ $CURRENT_ROOT_HASH == $ROOT_HASH ]]; then
echo "Already up2date"
exit 1
fi
else
if ! [[ $NO_DOWNLOAD ]]; then
cd "$MY_TMPDIR"
download_latest_json() {
JSON="${NAME}-latest.json"
curl ${BASEURL}/${JSON} --output ${JSON}
rm -f "/var/cache/${NAME}/${JSON}"
curl "${BASEURL}/${JSON}" --output /var/cache/${NAME}/${JSON}
ROOT_HASH="$(jq -r '.roothash' /var/cache/${NAME}/${JSON})"
VERSION="$(jq -r '.version' /var/cache/${NAME}/${JSON})"
mv "/var/cache/${NAME}/${JSON}" "/var/cache/${NAME}/${NAME}-${VERSION}.json"
JSON="/var/cache/${NAME}/${NAME}-${VERSION}.json"
curl "${BASEURL}/${NAME}-${VERSION}.json.sig" \
--output /var/cache/${NAME}/${NAME}-${VERSION}.json.sig
if ! openssl dgst -sha256 -verify /etc/pki/${NAME}/pubkey \
-signature /var/cache/${NAME}/${NAME}-${VERSION}.json.sig \
"/var/cache/${NAME}/${NAME}-${VERSION}.json"
then
rm -f "/var/cache/${NAME}/${NAME}-${VERSION}.json" \
"/var/cache/${NAME}/${NAME}-${VERSION}.json.sig"
return 1
fi
return 0
}
if [[ $USE_JSON ]]; then
JSON="${USE_JSON}"
else
JSON="$(realpath $1)"
cd ${JSON%/*}
download_latest_json
fi
IMAGE="$(jq -r '.name' ${JSON})-$(jq -r '.version' ${JSON})"
ROOT_HASH=$(jq -r '.roothash' ${JSON})
cd $MY_TMPDIR
if ! [[ $FORCE ]] && [[ $CURRENT_ROOT_HASH == $ROOT_HASH ]]; then
JSONDIR="${JSON%/*}"
ROOT_HASH="$(jq -r '.roothash' ${JSON})"
IMAGE_SIZE="$(jq -r '.imagesize' ${JSON})"
if ! [[ $FORCE ]] && ( \
[[ $CURRENT_ROOT_HASH == $ROOT_HASH ]] \
|| [[ -f /efi/EFI/${NAME}/bootx64-XXX-$ROOT_HASH.efi ]]
)
then
echo "Already up2date"
exit 1
exit 0
fi
if ! [[ $NO_DOWNLOAD ]]; then
[[ -d ${IMAGE} ]] || curl ${BASEURL}/${IMAGE}.tgz | tar xzf -
check_delta_size() {
local HASH="$1"
local TARGET_HASH="$2"
local SIZE NEW_HASH JSON NEW_SIZE
curl -s "${BASEURL}/${NAME}-${HASH}-delta.json" \
--output /var/cache/${NAME}/${NAME}-${HASH}-delta.json \
|| return -1
curl -s "${BASEURL}/${NAME}-${HASH}-delta.json.sig" \
--output /var/cache/${NAME}/${NAME}-${HASH}-delta.json.sig \
|| return -1
openssl dgst -sha256 -verify /etc/pki/${NAME}/pubkey \
-signature /var/cache/${NAME}/${NAME}-${HASH}-delta.json.sig \
/var/cache/${NAME}/${NAME}-${HASH}-delta.json \
&>/dev/null || return -1
JSON="/var/cache/${NAME}/${NAME}-${HASH}-delta.json"
SIZE="$(jq -r '.deltasize' $JSON)"
NEW_HASH="$(jq -r '.roothash' ${JSON})"
if [[ $NEW_HASH != $TARGET_HASH ]]; then
NEW_SIZE=$(check_delta_size "$NEW_HASH" "$TARGET_HASH")
[[ $? == -1 ]] && return -1
SIZE=$(($SIZE + $NEW_SIZE))
fi
echo $SIZE
return 0
}
download_delta_images() {
local HASH="$1"
local TARGET_HASH="$2"
local SIZE NEW_HASH NEW_SIZE
local JSON="/var/cache/${NAME}/${NAME}-${HASH}-delta.json"
curl -s "${BASEURL}/${NAME}-${HASH}-delta.img" \
--output /var/cache/${NAME}/${NAME}-${HASH}-delta.img \
|| return -1
jq -r '.deltasig' ${JSON} | xxd -r -p > "$MY_TMPDIR/deltasig"
openssl dgst -sha256 -verify /etc/pki/${NAME}/pubkey \
-signature "$MY_TMPDIR/deltasig" \
/var/cache/${NAME}/${NAME}-${HASH}-delta.img \
&>/dev/null || return -1
NEW_HASH="$(jq -r '.roothash' ${JSON})"
if [[ $NEW_HASH != $TARGET_HASH ]]; then
xdelta3 -c -d -s /dev/stdin /var/cache/${NAME}/${NAME}-${HASH}-delta.img \
| download_delta_images "$NEW_HASH" "$TARGET_HASH"
else
xdelta3 -c -d -s /dev/stdin /var/cache/${NAME}/${NAME}-${HASH}-delta.img
fi
}
[[ -d ${IMAGE} ]]
cd ${IMAGE}
unset FILES; declare -A FILES
while read _ file || [[ $file ]]; do
FILES["$file"]="1"
done < sha512sum.txt
if ! [[ $NO_CHECK ]]; then
# check integrity
openssl dgst -sha256 -verify "$sysroot"/etc/pki/${NAME}/pubkey \
-signature sha512sum.txt.sig sha512sum.txt
sha512sum --strict -c sha512sum.txt
for i in $(find . -type f); do
[[ $i == ./sha512sum.txt ]] && continue
[[ $i == ./sha512sum.txt.sig ]] && continue
if ! [[ ${FILES["$i"]} ]]; then
echo "File $i not signed"
exit 1
fi
done
fi
if [[ ${FILES["update.sh"]} ]] && [[ -e ./update.sh ]]; then
. ./update.sh
exit $?
fi
dd bs=4096 conv=fsync status=progress \
if=root.img \
if SIZE=$(check_delta_size "$CURRENT_ROOT_HASH" "$ROOT_HASH") && (($SIZE < $IMAGE_SIZE))
then
dd if=$CURRENT_ROOT_DEV bs=4096 count=$(($CURRENT_IMAGE_SIZE/4096)) \
| download_delta_images "$CURRENT_ROOT_HASH" "$ROOT_HASH" \
| dd bs=4096 conv=fsync status=progress \
of=${ROOT_DEV}-part${NEW_ROOT_PARTNO}
else
curl -C - "${BASEURL}/${NAME}-${ROOT_HASH}.img" \
| dd bs=4096 conv=fsync status=progress \
of=${ROOT_DEV}-part${NEW_ROOT_PARTNO}
fi
jq -r '.rootimgsig' ${JSON} | xxd -r -p > "$MY_TMPDIR/rootimgsig"
if ! dd bs=4096 \
if=${ROOT_DEV}-part${NEW_ROOT_PARTNO} \
count=$(($IMAGE_SIZE/4096)) \
| openssl dgst -sha256 -verify /etc/pki/${NAME}/pubkey \
-signature "$MY_TMPDIR/rootimgsig" /dev/stdin;
then
exit 1
fi
# set the new partition uuids
ROOT_UUID=${ROOT_HASH:32:8}-${ROOT_HASH:40:4}-${ROOT_HASH:44:4}-${ROOT_HASH:48:4}-${ROOT_HASH:52:12}
sfdisk --part-uuid ${ROOT_DEV} ${NEW_ROOT_PARTNO} ${ROOT_UUID}
jq -r '.efitarsig' ${JSON} | xxd -r -p > "$MY_TMPDIR/efitarsig"
curl -C - "${BASEURL}/${NAME}-${ROOT_HASH}-efi.tgz" \
--output "/var/cache/${NAME}/${NAME}-${ROOT_HASH}-efi.tgz"
if ! openssl dgst -sha256 -verify /etc/pki/${NAME}/pubkey \
-signature "$MY_TMPDIR/efitarsig" "${JSONDIR}/${NAME}-${ROOT_HASH}-efi.tgz";
then
rm -f "${JSONDIR}/${NAME}-${ROOT_HASH}-efi.tgz"
exit 1
fi
tar xzf "${JSONDIR}/${NAME}-${ROOT_HASH}-efi.tgz"
# install to /efi
if [[ -d efi/EFI ]]; then
cp -vr efi/EFI/* /efi/EFI/