242 lines
13 KiB
Bash
242 lines
13 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
#
|
||
|
|
# SPDX-License-Identifier: GPL-2.0
|
||
|
|
#
|
||
|
|
# Copyright (c) 2013-2023 Igor Pecovnik, igor@armbian.com
|
||
|
|
#
|
||
|
|
# This file is a part of the Armbian Build Framework
|
||
|
|
# https://github.com/armbian/build/
|
||
|
|
|
||
|
|
function create_new_rootfs_cache_tarball() {
|
||
|
|
# validate cache_fname is set
|
||
|
|
[[ -n "${cache_fname}" ]] || exit_with_error "create_new_rootfs_cache_tarball: cache_fname is not set"
|
||
|
|
# validate SDCARD is set
|
||
|
|
[[ -n "${SDCARD}" ]] || exit_with_error "create_new_rootfs_cache_tarball: SDCARD is not set"
|
||
|
|
# validate cache_name is set
|
||
|
|
[[ -n "${cache_name}" ]] || exit_with_error "create_new_rootfs_cache_tarball: cache_name is not set"
|
||
|
|
|
||
|
|
# create list of installed packages for debug purposes - this captures it's own stdout.
|
||
|
|
# @TODO: sanity check, compare this with the source of the hash coming from aggregation
|
||
|
|
chroot_sdcard "dpkg -l | grep ^ii | awk '{ print \$2\",\"\$3 }'" > "${cache_fname}.list"
|
||
|
|
echo "${AGGREGATED_ROOTFS_HASH_TEXT}" > "${cache_fname}.hash_text"
|
||
|
|
|
||
|
|
# Show the disk space usage of the rootfs
|
||
|
|
display_alert "Disk space usage of rootfs" "${RELEASE}:: ${cache_name}" "info"
|
||
|
|
run_host_command_logged "cd ${SDCARD} && " du -h -d 4 -x "." "| sort -h | tail -20"
|
||
|
|
wait_for_disk_sync "after disk-space usage report of rootfs"
|
||
|
|
|
||
|
|
declare compression_ratio_rootfs="${ROOTFS_COMPRESSION_RATIO:-"5"}"
|
||
|
|
|
||
|
|
display_alert "zstd tarball of rootfs" "${RELEASE}:: ${cache_name} :: compression ${compression_ratio_rootfs}" "info"
|
||
|
|
tar cp --xattrs --directory="$SDCARD"/ --exclude='./dev/*' --exclude='./proc/*' --exclude='./run/*' \
|
||
|
|
--exclude='./tmp/*' --exclude='./sys/*' --exclude='./home/*' --exclude='./root/*' . |
|
||
|
|
pv -p -b -r -s "$(du -sb "$SDCARD"/ | cut -f1)" -N "$(logging_echo_prefix_for_pv "store_rootfs") $cache_name" |
|
||
|
|
zstdmt "-${compression_ratio_rootfs}" -c > "${cache_fname}"
|
||
|
|
|
||
|
|
declare -a pv_tar_zstdmt_pipe_status=("${PIPESTATUS[@]}") # capture and the pipe_status array from PIPESTATUS
|
||
|
|
declare one_pipe_status
|
||
|
|
for one_pipe_status in "${pv_tar_zstdmt_pipe_status[@]}"; do
|
||
|
|
if [[ "$one_pipe_status" != "0" ]]; then
|
||
|
|
exit_with_error "create_new_rootfs_cache_tarball: compress: ${cache_fname} failed (${pv_tar_zstdmt_pipe_status[*]}) - out of disk space?"
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
wait_for_disk_sync "after zstd tarball rootfs"
|
||
|
|
|
||
|
|
# get the human readable size of the cache
|
||
|
|
local cache_size
|
||
|
|
cache_size=$(du -sh "${cache_fname}" | cut -f1)
|
||
|
|
|
||
|
|
display_alert "rootfs cache created" "${cache_fname} [${cache_size}]" "info"
|
||
|
|
}
|
||
|
|
|
||
|
|
function create_new_rootfs_cache_via_debootstrap() {
|
||
|
|
[[ ! -d "${SDCARD:?}" ]] && exit_with_error "create_new_rootfs_cache_via_debootstrap: ${SDCARD} is not a directory"
|
||
|
|
|
||
|
|
# this is different between debootstrap and regular apt-get; here we use acng as a prefix to the real repo
|
||
|
|
declare debootstrap_apt_mirror="http://${APT_MIRROR}"
|
||
|
|
if [[ "${MANAGE_ACNG}" == "yes" ]]; then
|
||
|
|
local debootstrap_apt_mirror="http://localhost:3142/${APT_MIRROR}"
|
||
|
|
acng_check_status_or_restart
|
||
|
|
fi
|
||
|
|
|
||
|
|
# @TODO: one day: https://gitlab.mister-muffin.de/josch/mmdebstrap/src/branch/main/mmdebstrap
|
||
|
|
|
||
|
|
display_alert "Installing base system with ${#AGGREGATED_PACKAGES_DEBOOTSTRAP[@]} packages" "Stage 1/2" "info"
|
||
|
|
cd "${SDCARD}" || exit_with_error "cray-cray about SDCARD" "${SDCARD}" # this will prevent error sh: 0: getcwd() failed
|
||
|
|
|
||
|
|
declare -a deboostrap_arguments=(
|
||
|
|
"--variant=minbase" # minimal base variant. go ask Debian about it.
|
||
|
|
"--arch=${ARCH}" # the arch
|
||
|
|
"'--include=${AGGREGATED_PACKAGES_DEBOOTSTRAP_COMMA}'" # from aggregation.py
|
||
|
|
"'--components=${AGGREGATED_DEBOOTSTRAP_COMPONENTS_COMMA}'" # from aggregation.py
|
||
|
|
)
|
||
|
|
|
||
|
|
# Small detour for local apt caching option.
|
||
|
|
local_apt_deb_cache_prepare "before debootstrap" # sets LOCAL_APT_CACHE_INFO
|
||
|
|
if [[ "${LOCAL_APT_CACHE_INFO[USE]}" == "yes" ]]; then
|
||
|
|
deboostrap_arguments+=("--cache-dir=${LOCAL_APT_CACHE_INFO[HOST_DEBOOTSTRAP_CACHE_DIR]}") # cache .deb's used
|
||
|
|
fi
|
||
|
|
|
||
|
|
deboostrap_arguments+=("--foreign") # release name
|
||
|
|
|
||
|
|
deboostrap_arguments+=("${RELEASE}" "${SDCARD}/" "${debootstrap_apt_mirror}") # release, path and mirror; always last, positional arguments.
|
||
|
|
|
||
|
|
run_host_command_logged debootstrap "${deboostrap_arguments[@]}" || {
|
||
|
|
exit_with_error "Debootstrap first stage failed" "${RELEASE} ${DESKTOP_APPGROUPS_SELECTED} ${DESKTOP_ENVIRONMENT} ${BUILD_MINIMAL}"
|
||
|
|
}
|
||
|
|
[[ ! -f ${SDCARD}/debootstrap/debootstrap ]] && exit_with_error "Debootstrap first stage did not produce marker file"
|
||
|
|
|
||
|
|
skip_target_check="yes" local_apt_deb_cache_prepare "after debootstrap first stage" # just for size reference in logs; skip the target check: debootstrap uses it for second stage.
|
||
|
|
|
||
|
|
deploy_qemu_binary_to_chroot "${SDCARD}" # this is cleaned-up later by post_debootstrap_tweaks() @TODO: which is too late for a cache
|
||
|
|
|
||
|
|
display_alert "Installing base system" "Stage 2/2" "info"
|
||
|
|
declare -g if_error_detail_message="Debootstrap second stage failed ${RELEASE} ${DESKTOP_APPGROUPS_SELECTED} ${DESKTOP_ENVIRONMENT} ${BUILD_MINIMAL}"
|
||
|
|
chroot_sdcard LC_ALL=C LANG=C /debootstrap/debootstrap --second-stage
|
||
|
|
[[ ! -f "${SDCARD}/bin/bash" ]] && exit_with_error "Debootstrap first stage did not produce /bin/bash"
|
||
|
|
|
||
|
|
# Done with debootstrap. Clean-up it's litterbox.
|
||
|
|
display_alert "Cleaning up after debootstrap" "debootstrap cleanup" "info"
|
||
|
|
run_host_command_logged rm -rf "${SDCARD}/var/cache/apt" "${SDCARD}/var/lib/apt/lists"
|
||
|
|
|
||
|
|
local_apt_deb_cache_prepare "after debootstrap second stage" # just for size reference in logs
|
||
|
|
|
||
|
|
mount_chroot "${SDCARD}" # we mount the chroot here... it's un-mounted below when all is done, or by cleanup handler '' @TODO
|
||
|
|
|
||
|
|
display_alert "Diverting" "initctl/start-stop-daemon" "info"
|
||
|
|
# policy-rc.d script prevents starting or reloading services during image creation
|
||
|
|
printf '#!/bin/sh\nexit 101' > $SDCARD/usr/sbin/policy-rc.d
|
||
|
|
chroot_sdcard LC_ALL=C LANG=C dpkg-divert --quiet --local --rename --add /sbin/initctl
|
||
|
|
chroot_sdcard LC_ALL=C LANG=C dpkg-divert --quiet --local --rename --add /sbin/start-stop-daemon
|
||
|
|
printf '#!/bin/sh\necho "Warning: Fake start-stop-daemon called, doing nothing"' > "$SDCARD/sbin/start-stop-daemon"
|
||
|
|
printf '#!/bin/sh\necho "Warning: Fake initctl called, doing nothing"' > "$SDCARD/sbin/initctl"
|
||
|
|
chmod 755 "${SDCARD}/usr/sbin/policy-rc.d"
|
||
|
|
chmod 755 "${SDCARD}/sbin/initctl"
|
||
|
|
chmod 755 "${SDCARD}/sbin/start-stop-daemon"
|
||
|
|
|
||
|
|
# stage: configure language and locales.
|
||
|
|
# this _requires_ DEST_LANG, otherwise, bomb: if it's not here _all_ locales will be generated which is very slow.
|
||
|
|
display_alert "Configuring locales" "DEST_LANG: ${DEST_LANG}" "info"
|
||
|
|
[[ "x${DEST_LANG}x" == "xx" ]] && exit_with_error "Bug: got to config locales without DEST_LANG set"
|
||
|
|
|
||
|
|
[[ -f $SDCARD/etc/locale.gen ]] && sed -i "s/^# ${DEST_LANG}/${DEST_LANG}/" $SDCARD/etc/locale.gen
|
||
|
|
chroot_sdcard LC_ALL=C LANG=C locale-gen "${DEST_LANG}"
|
||
|
|
chroot_sdcard LC_ALL=C LANG=C update-locale "LANG=${DEST_LANG}" "LANGUAGE=${DEST_LANG}" "LC_MESSAGES=${DEST_LANG}"
|
||
|
|
|
||
|
|
if [[ -f $SDCARD/etc/default/console-setup ]]; then
|
||
|
|
# @TODO: Should be configurable.
|
||
|
|
sed -e 's/CHARMAP=.*/CHARMAP="UTF-8"/' -e 's/FONTSIZE=.*/FONTSIZE="8x16"/' \
|
||
|
|
-e 's/CODESET=.*/CODESET="guess"/' -i "$SDCARD/etc/default/console-setup"
|
||
|
|
chroot_sdcard LC_ALL=C LANG=C setupcon --save --force
|
||
|
|
fi
|
||
|
|
|
||
|
|
# stage: create apt-get sources list (basic Debian/Ubuntu apt sources, no external nor PPAS).
|
||
|
|
# for the Armbian repo, only the components which are _not_ produced by armbian/build are included (-desktop and -utils)
|
||
|
|
create_sources_list_and_deploy_repo_key "root" "$RELEASE" "$SDCARD/"
|
||
|
|
|
||
|
|
# optionally add armhf arhitecture to arm64, if asked to do so.
|
||
|
|
if [[ "a${ARMHF_ARCH}" == "ayes" ]]; then
|
||
|
|
[[ $ARCH == arm64 ]] && chroot_sdcard LC_ALL=C LANG=C dpkg --add-architecture armhf
|
||
|
|
fi
|
||
|
|
|
||
|
|
# this should fix resolvconf installation failure in some cases
|
||
|
|
chroot_sdcard 'echo "resolvconf resolvconf/linkify-resolvconf boolean false" | debconf-set-selections'
|
||
|
|
|
||
|
|
# Add external / PPAs to apt sources; decides internally based on minimal/cli/desktop dir/file structure
|
||
|
|
add_apt_sources
|
||
|
|
|
||
|
|
# @TODO: use asset logging for this; actually log contents of the files too
|
||
|
|
run_host_command_logged ls -l "${SDCARD}/usr/share/keyrings"
|
||
|
|
run_host_command_logged ls -l "${SDCARD}/etc/apt/sources.list.d"
|
||
|
|
run_host_command_logged cat "${SDCARD}/etc/apt/sources.list"
|
||
|
|
|
||
|
|
# stage: update packages list
|
||
|
|
display_alert "Updating package list" "$RELEASE" "info"
|
||
|
|
do_with_retries 3 chroot_sdcard_apt_get_update
|
||
|
|
|
||
|
|
# stage: upgrade base packages from xxx-updates and xxx-backports repository branches
|
||
|
|
display_alert "Upgrading base packages" "Armbian" "info"
|
||
|
|
do_with_retries 3 chroot_sdcard_apt_get upgrade
|
||
|
|
|
||
|
|
# stage: install additional packages
|
||
|
|
display_alert "Installing the main packages for" "Armbian" "info"
|
||
|
|
declare -g if_error_detail_message="Installation of Armbian main packages for ${RELEASE} ${DESKTOP_APPGROUPS_SELECTED} ${DESKTOP_ENVIRONMENT} ${BUILD_MINIMAL} failed"
|
||
|
|
# First, try to download-only up to 3 times, to work around network/proxy problems.
|
||
|
|
# AGGREGATED_PACKAGES_ROOTFS is generated by aggregation.py
|
||
|
|
chroot_sdcard_apt_get_install_dry_run "${AGGREGATED_PACKAGES_ROOTFS[@]}"
|
||
|
|
do_with_retries 3 chroot_sdcard_apt_get_install_download_only "${AGGREGATED_PACKAGES_ROOTFS[@]}"
|
||
|
|
|
||
|
|
# Now do the install, all packages should have been downloaded by now
|
||
|
|
chroot_sdcard_apt_get_install "${AGGREGATED_PACKAGES_ROOTFS[@]}"
|
||
|
|
|
||
|
|
if [[ $BUILD_DESKTOP == "yes" ]]; then
|
||
|
|
# how how many items in AGGREGATED_PACKAGES_DESKTOP array
|
||
|
|
display_alert "Installing ${#AGGREGATED_PACKAGES_DESKTOP[@]} desktop packages" "${RELEASE} ${DESKTOP_ENVIRONMENT}" "info"
|
||
|
|
|
||
|
|
# dry-run, make sure everything can be installed.
|
||
|
|
chroot_sdcard_apt_get_install_dry_run "${AGGREGATED_PACKAGES_DESKTOP[@]}"
|
||
|
|
|
||
|
|
# Retry download-only 3 times first.
|
||
|
|
do_with_retries 3 chroot_sdcard_apt_get_install_download_only "${AGGREGATED_PACKAGES_DESKTOP[@]}"
|
||
|
|
|
||
|
|
# Then do the actual install.
|
||
|
|
declare -g if_error_detail_message="Installation of Armbian desktop packages for ${RELEASE} ${DESKTOP_APPGROUPS_SELECTED} ${DESKTOP_ENVIRONMENT} ${BUILD_MINIMAL} failed"
|
||
|
|
chroot_sdcard_apt_get_install "${AGGREGATED_PACKAGES_DESKTOP[@]}"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# stage: check md5 sum of installed packages. Just in case. @TODO: rpardini: this should also be done when a cache is used, not only when it is created
|
||
|
|
display_alert "Checking MD5 sum of installed packages" "debsums" "info"
|
||
|
|
declare -g if_error_detail_message="Check MD5 sum of installed packages failed"
|
||
|
|
chroot_sdcard debsums --silent
|
||
|
|
|
||
|
|
# # Remove packages from packages.uninstall
|
||
|
|
# # @TODO: aggregation.py handling of this... if we wanted it removed in rootfs cache, why did we install it in the first place?
|
||
|
|
# display_alert "Uninstall packages" "$PACKAGE_LIST_UNINSTALL" "info"
|
||
|
|
# # shellcheck disable=SC2086
|
||
|
|
# DONT_MAINTAIN_APT_CACHE="yes" chroot_sdcard_apt_get purge $PACKAGE_LIST_UNINSTALL
|
||
|
|
|
||
|
|
# # if we remove with --purge then this is not needed
|
||
|
|
# # stage: purge residual packages
|
||
|
|
# display_alert "Purging residual packages for" "Armbian" "info"
|
||
|
|
# PURGINGPACKAGES=$(chroot $SDCARD /bin/bash -c "dpkg -l | grep \"^rc\" | awk '{print \$2}' | tr \"\n\" \" \"")
|
||
|
|
# DONT_MAINTAIN_APT_CACHE="yes" chroot_sdcard_apt_get purge $PURGINGPACKAGES
|
||
|
|
|
||
|
|
# stage: remove packages that are installed, but not required anymore after other packages were installed/removed.
|
||
|
|
# don't touch the local cache.
|
||
|
|
DONT_MAINTAIN_APT_CACHE="yes" chroot_sdcard_apt_get autoremove
|
||
|
|
|
||
|
|
# Purge/clean apt cache in the target. It should _not_ have been used, but if it was, warn & clean.
|
||
|
|
apt_purge_unneeded_packages_and_clean_apt_caches
|
||
|
|
|
||
|
|
# DEBUG: print free space
|
||
|
|
local free_space
|
||
|
|
free_space=$(LC_ALL=C df -h)
|
||
|
|
display_alert "Free disk space on rootfs" "SDCARD: $(echo -e "${free_space}" | awk -v mp="${SDCARD}" '$6==mp {print $5}')" "info"
|
||
|
|
|
||
|
|
# this is needed for the build process later since resolvconf generated file in /run is not saved
|
||
|
|
run_host_command_logged rm -v "${SDCARD}"/etc/resolv.conf
|
||
|
|
run_host_command_logged echo "nameserver $NAMESERVER" ">" "${SDCARD}"/etc/resolv.conf
|
||
|
|
|
||
|
|
# Remove `machine-id` (https://www.freedesktop.org/software/systemd/man/machine-id.html)
|
||
|
|
# Note: This will mark machine `firstboot`
|
||
|
|
run_host_command_logged echo "uninitialized" ">" "${SDCARD}/etc/machine-id"
|
||
|
|
run_host_command_logged rm -v "${SDCARD}/var/lib/dbus/machine-id"
|
||
|
|
|
||
|
|
# Mask `systemd-firstboot.service` which will prompt locale, timezone and root-password too early.
|
||
|
|
# `armbian-first-run` will do the same thing later
|
||
|
|
chroot_sdcard systemctl mask systemd-firstboot.service
|
||
|
|
|
||
|
|
# stage: make rootfs cache archive
|
||
|
|
display_alert "Ending debootstrap process and preparing cache" "$RELEASE" "info"
|
||
|
|
wait_for_disk_sync "before tar rootfs"
|
||
|
|
|
||
|
|
# we're done with using the chroot which we mounted above.
|
||
|
|
# if something failed, the cleanup handler (via trap manager) will take care of it.
|
||
|
|
umount_chroot "${SDCARD}"
|
||
|
|
|
||
|
|
wait_for_disk_sync "after unmounting chroot used for rootfs build"
|
||
|
|
|
||
|
|
return 0
|
||
|
|
}
|