271 lines
13 KiB
Bash
271 lines
13 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
#
|
||
|
|
# SPDX-License-Identifier: GPL-2.0
|
||
|
|
#
|
||
|
|
# Copyright (c) 2013-2023 Igor Pecovnik, igor@armbian.com
|
||
|
|
#
|
||
|
|
# This file is a part of the Armbian Build Framework
|
||
|
|
# https://github.com/armbian/build/
|
||
|
|
|
||
|
|
# This is not run under logging section.
|
||
|
|
function kernel_prepare_bare_repo_decide_shallow_or_full() {
|
||
|
|
declare decision="not_yet_decided" decision_why="unknown"
|
||
|
|
declare -i ask_for_user_confirmation=0
|
||
|
|
declare cache_git_bare_dir="${SRC}/cache/git-bare"
|
||
|
|
declare FULL_kernel_git_bare_tree="${cache_git_bare_dir}/kernel" # for reuse below
|
||
|
|
declare FULL_marker="git-bare/kernel/" # part of path of the full version; "magic string"
|
||
|
|
declare SHALLOW_kernel_git_bare_tree="${cache_git_bare_dir}/shallow-kernel-${KERNEL_MAJOR_MINOR}" # for reuse below
|
||
|
|
git_bundles_dir="${SRC}/cache/git-bundles/kernel" # set outer scope variable
|
||
|
|
|
||
|
|
run_host_command_logged mkdir -p "${cache_git_bare_dir}"
|
||
|
|
|
||
|
|
# We've two options here.
|
||
|
|
# One is to use the full version, which is around 3gb. "linux-complete.git.tar" and _always_ useful
|
||
|
|
# The other is to use the shallow version, which is around 300mb. "linux-shallow-6.1.git.tar". Not always useful, might end up unshallowing it later in some fetch.
|
||
|
|
# How to decide which one to use?
|
||
|
|
# @TODO: Zero: If on CI=true (Github Actions, Gitlab CI, etc): we wanna use shallow for hosted runners, and full for self-hosted runners.
|
||
|
|
# First: if the working copy already exists and is valid: use whatever that used before.
|
||
|
|
# Second: if the full version is already there, use it. It's the most complete, and serves all purposes.
|
||
|
|
# Third: if the shallow version is already there, for the KERNEL_MAJOR_MINOR in question, use it.
|
||
|
|
# Fourth: if none is there; decide based on common sense. The shallow tree is useful until we fetch something that makes it unshallow.
|
||
|
|
# - vendor kernels (looking at you, rockchip-rk3588) are a mess and will cause fetches to unshallow?
|
||
|
|
# - if free disk space on target is less than 32gb (magic number?) use the shallow. It won't fit otherwise.
|
||
|
|
# - if the target resides on `mmc` device, use the shallow. It's too much disk traffic otherwise.
|
||
|
|
# - TODO: might be we don't carry a shallow gitball in ghcr.io for the wanted version -- how to know?
|
||
|
|
|
||
|
|
# validate kernel_work_dir is set
|
||
|
|
[[ -z "${kernel_work_dir}" ]] && exit_with_error "kernel_work_dir is not set"
|
||
|
|
[[ -z "${bare_tree_done_marker_file}" ]] && exit_with_error "bare_tree_done_marker_file is not set"
|
||
|
|
|
||
|
|
# gather info about the storage device on which the bare tree would, or already does, reside
|
||
|
|
# @TODO refactor this out of here
|
||
|
|
# Find the type of device (mmc/nvme/scsi/etc) (not type of filesystem...) that ${cache_git_bare_dir} resides on.
|
||
|
|
declare device_backing_dir
|
||
|
|
device_backing_dir="$(findmnt --nofsroot -n -o SOURCE --target "${cache_git_bare_dir}")"
|
||
|
|
declare -i device_backing_dir_is_slow=0
|
||
|
|
# if device_backing_dir contains "mmc", set device_backing_dir_is_slow=1
|
||
|
|
if [[ "${device_backing_dir}" == *"mmc"* ]]; then
|
||
|
|
display_alert "Device backing ${cache_git_bare_dir} is eMMC/SD/etc" "${device_backing_dir}" "git"
|
||
|
|
device_backing_dir_is_slow=1
|
||
|
|
fi
|
||
|
|
display_alert "Device backing ${cache_git_bare_dir}" "${device_backing_dir}" "git"
|
||
|
|
|
||
|
|
# @TODO Refactor this out of here - many other places using similar
|
||
|
|
# Get the free disk space for the ${cache_git_bare_dir}, in MiB.
|
||
|
|
declare -i free_space_mib
|
||
|
|
free_space_mib="$(df -BM --output=avail "${cache_git_bare_dir}" | tail -n 1 | sed 's/M//')"
|
||
|
|
display_alert "Free space on ${cache_git_bare_dir}" "${free_space_mib} MiB" "git"
|
||
|
|
|
||
|
|
if [[ "${decision}" == "not_yet_decided" ]]; then
|
||
|
|
# First: if ${kernel_work_dir}/.git already exists, use whatever that used before, by reading its .git file.
|
||
|
|
if [[ -f "${kernel_work_dir}/.git" ]]; then
|
||
|
|
display_alert "Worktree .git file already exists" "${kernel_work_dir}/.git" "git"
|
||
|
|
if grep -q "${FULL_marker}" "${kernel_work_dir}/.git"; then
|
||
|
|
display_alert "Worktree .git file indicates full version bare" "${kernel_work_dir}/.git" "git"
|
||
|
|
decision="full"
|
||
|
|
decision_why="existing worktree points to full"
|
||
|
|
else
|
||
|
|
display_alert "Worktree .git file does NOT indicate full bare" "${kernel_work_dir}/.git" "git"
|
||
|
|
decision="shallow"
|
||
|
|
decision_why="existing worktree points to shallow"
|
||
|
|
fi
|
||
|
|
else
|
||
|
|
display_alert "Worktree .git file does NOT exist" "${kernel_work_dir}/.git" "git"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [[ "${decision}" == "not_yet_decided" ]]; then
|
||
|
|
# Second: if ${FULL_kernel_git_bare_tree} and ${FULL_kernel_git_bare_tree}/${bare_tree_done_marker_file} exists
|
||
|
|
if [[ -d "${FULL_kernel_git_bare_tree}" && -f "${FULL_kernel_git_bare_tree}/${bare_tree_done_marker_file}" ]]; then
|
||
|
|
display_alert "Full version bare tree exists and is ready to go" "${FULL_kernel_git_bare_tree}" "git"
|
||
|
|
decision="full" # end of story
|
||
|
|
decision_why="full version bare tree exists and is ready to go"
|
||
|
|
else
|
||
|
|
display_alert "Full version bare tree does NOT exist or is NOT ready to go" "${FULL_kernel_git_bare_tree}" "git"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
# simplest, via parameter/env var, something like KERNEL_GIT=shallow or KERNEL_GIT=full to force.
|
||
|
|
declare forced_decision="${KERNEL_GIT:-"none"}"
|
||
|
|
if [[ "${decision}" == "not_yet_decided" ]]; then
|
||
|
|
case "${forced_decision,,}" in # lowercase
|
||
|
|
shallow)
|
||
|
|
display_alert "Forced shallow via" "KERNEL_GIT=shallow" "git"
|
||
|
|
decision="shallow"
|
||
|
|
decision_why="forced by KERNEL_GIT=shallow"
|
||
|
|
;;
|
||
|
|
|
||
|
|
full)
|
||
|
|
display_alert "Forced full via" "KERNEL_GIT=full" "git"
|
||
|
|
decision="full"
|
||
|
|
decision_why="forced by KERNEL_GIT=full"
|
||
|
|
;;
|
||
|
|
esac
|
||
|
|
elif [[ "${forced_decision}" != "none" && "${forced_decision}" != "${decision}" ]]; then
|
||
|
|
display_alert "Can't change Kernel git from '${decision}' to '${forced_decision}'" "${decision_why}" "warn"
|
||
|
|
countdown_and_continue_if_not_aborted 3
|
||
|
|
fi
|
||
|
|
|
||
|
|
# @TODO: using shallow for vendor kernels (eg rockchip-rk3588) is a bad idea. It's a mess and will cause fetches to unshallow.
|
||
|
|
# So skip Third if family indicates so, via KERNEL_VENDOR_DO_NOT_SHALLOW=yes in config.
|
||
|
|
|
||
|
|
if [[ "${decision}" == "not_yet_decided" ]]; then
|
||
|
|
# Third: if ${SHALLOW_kernel_git_bare_tree} and ${SHALLOW_kernel_git_bare_tree}/${bare_tree_done_marker_file} exists
|
||
|
|
if [[ -d "${SHALLOW_kernel_git_bare_tree}" && -f "${SHALLOW_kernel_git_bare_tree}/${bare_tree_done_marker_file}" ]]; then
|
||
|
|
display_alert "Shallow bare tree for {KERNEL_MAJOR_MINOR} exists and is ready to go" "${SHALLOW_kernel_git_bare_tree}" "git"
|
||
|
|
decision="shallow" # end of story
|
||
|
|
decision_why="shallow ${KERNEL_MAJOR_MINOR} bare tree exists and is ready to go"
|
||
|
|
else
|
||
|
|
display_alert "Shallow bare tree for ${KERNEL_MAJOR_MINOR} does NOT exist or is NOT ready to go" "${SHALLOW_kernel_git_bare_tree}" "git"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [[ "${decision}" == "not_yet_decided" ]]; then
|
||
|
|
ask_for_user_confirmation=1 # no tree exists, will be the first time. offer option for user to abort.
|
||
|
|
|
||
|
|
# TODO "magic number" here, make configurable
|
||
|
|
if [[ ${free_space_mib} -lt 32768 ]] || [[ ${device_backing_dir_is_slow} -eq 1 ]]; then
|
||
|
|
decision="shallow"
|
||
|
|
decision_why="slow storage device (${device_backing_dir}) or low disk space (${free_space_mib} MiB)"
|
||
|
|
else
|
||
|
|
decision="full"
|
||
|
|
decision_why="fast storage device (${device_backing_dir}) and enough disk space (${free_space_mib} MiB)"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
display_alert "Using ${decision} Kernel bare tree for ${KERNEL_MAJOR_MINOR}" "${decision_why}" "info"
|
||
|
|
|
||
|
|
declare base_oras_ref="${GHCR_SOURCE}/armbian/shallow" # @TODO allow changing this
|
||
|
|
declare estimated_dl_size_mib=0 benefits="" cons=""
|
||
|
|
case "${decision}" in
|
||
|
|
shallow)
|
||
|
|
# Outer scope variables
|
||
|
|
kernel_git_bare_tree="${SHALLOW_kernel_git_bare_tree}"
|
||
|
|
git_kernel_ball_fn="linux-shallow-${KERNEL_MAJOR_MINOR}.git.tar"
|
||
|
|
git_kernel_oras_ref="${base_oras_ref}/kernel-git-shallow-${KERNEL_MAJOR_MINOR}:latest"
|
||
|
|
estimated_dl_size_mib=300
|
||
|
|
benefits="smaller download, less disk space consumed"
|
||
|
|
cons="less useful over time, no history, not shared across versions"
|
||
|
|
;;
|
||
|
|
|
||
|
|
*)
|
||
|
|
# Outer scope variables
|
||
|
|
kernel_git_bare_tree="${FULL_kernel_git_bare_tree}" # sets the outer scope variable
|
||
|
|
git_kernel_ball_fn="linux-complete.git.tar"
|
||
|
|
git_kernel_oras_ref="${base_oras_ref}/kernel-git:latest"
|
||
|
|
estimated_dl_size_mib=2700
|
||
|
|
benefits="more useful over time, full history, single tree across all versions"
|
||
|
|
cons="bigger download, more disk space consumed"
|
||
|
|
;;
|
||
|
|
esac
|
||
|
|
|
||
|
|
display_alert "kernel_git_bare_tree" "${kernel_git_bare_tree}" "git"
|
||
|
|
display_alert "git_kernel_ball_fn" "${git_kernel_ball_fn}" "git"
|
||
|
|
display_alert "git_kernel_oras_ref" "${git_kernel_oras_ref}" "git"
|
||
|
|
display_alert "estimated_dl_size_mib" "${estimated_dl_size_mib}" "git"
|
||
|
|
|
||
|
|
if [[ ${ask_for_user_confirmation} -eq 1 && -t 0 ]]; then # -t 0 means stdin is a terminal
|
||
|
|
echo "--------------------------------------------------------------------------------------------------------------------" >&2
|
||
|
|
display_alert "Warning: no Kernel bare tree exists for version ${KERNEL_MAJOR_MINOR} - about to start downloading." "" "wrn"
|
||
|
|
display_alert "Armbian is going to use a '${decision}' git tree, which is around" "${estimated_dl_size_mib}MiB" ""
|
||
|
|
display_alert "This was decided due to" "${decision_why}" ""
|
||
|
|
display_alert "Benefits of using a '${decision}' git tree" "${benefits}" "info"
|
||
|
|
display_alert "Cons of using a '${decision}' git tree" "${cons}" "wrn"
|
||
|
|
display_alert "You can abort now, and pass either KERNEL_GIT=full or KERNEL_GIT=shallow to force." "" "wrn"
|
||
|
|
display_alert "If you want to abort" "press Ctrl+C before the countdown ends in 60 seconds" "wrn"
|
||
|
|
display_alert "If you agree with the decision to use a '${decision}' git tree" "press any other key to continue" ""
|
||
|
|
countdown_and_continue_if_not_aborted 60
|
||
|
|
fi
|
||
|
|
|
||
|
|
return 0
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
# This is run under the logging section, so, no interactive parts -- please.
|
||
|
|
function kernel_prepare_bare_repo_from_oras_gitball() {
|
||
|
|
|
||
|
|
# validate kernel_git_bare_tree and bare_tree_done_marker_file are set
|
||
|
|
if [[ -z "${kernel_git_bare_tree}" ]]; then
|
||
|
|
exit_with_error "kernel_git_bare_tree is not set"
|
||
|
|
fi
|
||
|
|
if [[ -z "${bare_tree_done_marker_file}" ]]; then
|
||
|
|
exit_with_error "bare_tree_done_marker_file is not set"
|
||
|
|
fi
|
||
|
|
|
||
|
|
declare kernel_git_bare_tree_done_marker="${kernel_git_bare_tree}/${bare_tree_done_marker_file}"
|
||
|
|
|
||
|
|
if [[ ! -d "${kernel_git_bare_tree}" || ! -f "${kernel_git_bare_tree_done_marker}" ]]; then
|
||
|
|
display_alert "Preparing bare kernel git tree" "this might take a long time" "info"
|
||
|
|
|
||
|
|
if [[ -d "${kernel_git_bare_tree}" ]]; then
|
||
|
|
display_alert "Removing old kernel bare tree" "${kernel_git_bare_tree}" "info"
|
||
|
|
run_host_command_logged rm -rf "${kernel_git_bare_tree}"
|
||
|
|
fi
|
||
|
|
|
||
|
|
wait_for_disk_sync "before Kernel git tree download"
|
||
|
|
|
||
|
|
# now, make sure we've the bundle downloaded correctly...
|
||
|
|
# this defines linux_kernel_clone_bundle_file
|
||
|
|
declare linux_kernel_clone_tar_file
|
||
|
|
download_git_kernel_gitball_via_oras # sets linux_kernel_clone_tar_file or dies
|
||
|
|
|
||
|
|
wait_for_disk_sync "before Kernel git extraction"
|
||
|
|
|
||
|
|
# Just extract the tar_file into the "${kernel_git_bare_tree}" directory, no further work needed.
|
||
|
|
run_host_command_logged mkdir -p "${kernel_git_bare_tree}"
|
||
|
|
# @TODO chance of a pv thingy here?
|
||
|
|
run_host_command_logged tar -xf "${linux_kernel_clone_tar_file}" -C "${kernel_git_bare_tree}"
|
||
|
|
|
||
|
|
wait_for_disk_sync "after Kernel git extraction"
|
||
|
|
|
||
|
|
# sanity check
|
||
|
|
if [[ ! -d "${kernel_git_bare_tree}/.git" ]]; then
|
||
|
|
exit_with_error "Kernel bare tree is missing .git directory ${kernel_git_bare_tree}"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# write the marker file
|
||
|
|
touch "${kernel_git_bare_tree_done_marker}"
|
||
|
|
else
|
||
|
|
display_alert "Kernel bare tree already exists" "${kernel_git_bare_tree}" "cachehit"
|
||
|
|
fi
|
||
|
|
|
||
|
|
git_ensure_safe_directory "${kernel_git_bare_tree}"
|
||
|
|
|
||
|
|
return 0
|
||
|
|
}
|
||
|
|
|
||
|
|
function download_git_kernel_gitball_via_oras() {
|
||
|
|
# validate git_bundles_dir, git_kernel_ball_fn, and git_kernel_oras_ref are set
|
||
|
|
if [[ -z "${git_bundles_dir}" ]]; then
|
||
|
|
exit_with_error "git_bundles_dir is not set"
|
||
|
|
fi
|
||
|
|
if [[ -z "${git_kernel_ball_fn}" ]]; then
|
||
|
|
exit_with_error "git_kernel_ball_fn is not set"
|
||
|
|
fi
|
||
|
|
if [[ -z "${git_kernel_oras_ref}" ]]; then
|
||
|
|
exit_with_error "git_kernel_oras_ref is not set"
|
||
|
|
fi
|
||
|
|
|
||
|
|
run_host_command_logged mkdir -p "${git_bundles_dir}" # this is later cleaned up by kernel_cleanup_bundle_artifacts()
|
||
|
|
|
||
|
|
# defines outer scope value
|
||
|
|
linux_kernel_clone_tar_file="${git_bundles_dir}/${git_kernel_ball_fn}"
|
||
|
|
|
||
|
|
# if the file already exists, do nothing; it will only exist if successfully downloaded by ORAS
|
||
|
|
if [[ -f "${linux_kernel_clone_tar_file}" ]]; then
|
||
|
|
display_alert "Kernel git-tarball already exists" "${git_kernel_ball_fn}" "cachehit"
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
# do_with_retries 5 xxx ? -- no -- oras_pull_artifact_file should do it's own retries.
|
||
|
|
oras_pull_artifact_file "${git_kernel_oras_ref}" "${git_bundles_dir}" "${git_kernel_ball_fn}"
|
||
|
|
|
||
|
|
# sanity check
|
||
|
|
if [[ ! -f "${linux_kernel_clone_tar_file}" ]]; then
|
||
|
|
exit_with_error "Kernel git-tarball download failed ${linux_kernel_clone_tar_file}"
|
||
|
|
fi
|
||
|
|
|
||
|
|
return 0
|
||
|
|
|
||
|
|
}
|