build/lib/functions/general/python-tools.sh

155 lines
6.9 KiB
Bash
Raw Normal View History

#!/usr/bin/env bash
#
# SPDX-License-Identifier: GPL-2.0
#
# Copyright (c) 2013-2023 Igor Pecovnik, igor@armbian.com
#
# This file is a part of the Armbian Build Framework
# https://github.com/armbian/build/
# This whole thing is a big "I refuse to use venv in a simple bash script" delusion.
# If you know to tame it, teach me. I'd rather not know about PYTHONUSERBASE and such.
# --rpardini
function early_prepare_pip3_dependencies_for_python_tools() {
# This is like a stupid version of requirements.txt
declare -a -g python3_pip_dependencies=(
"unidiff==0.7.4" # for parsing unified diff
"GitPython==3.1.30" # for manipulating git repos
"unidecode==1.3.6" # for converting strings to ascii
"coloredlogs==15.0.1" # for colored logging
"PyYAML==6.0" # for parsing/writing YAML
"oras==0.1.17" # for OCI stuff in mapper-oci-update
"Jinja2==3.1.2" # for templating
"rich==13.4.1" # for rich text formatting
)
return 0
}
# call: prepare_python_and_pip # this defines global PYTHON3_INFO dict and PYTHON3_VARS array
function prepare_python_and_pip() {
assert_prepared_host # this needs a prepared host to work; avoid fake errors about "python3-pip" not being installed
# First determine with python3 to use; requires knowing the HOSTRELEASE.
[[ -z "${HOSTRELEASE}" ]] && exit_with_error "HOSTRELEASE is not set"
# fake-memoize this, it's expensive and does not need to be done twice
declare -g _already_prepared_python_and_pip="${_already_prepared_python_and_pip:-no}"
if [[ "${_already_prepared_python_and_pip}" == "yes" ]]; then
display_alert "All Python preparation done before" "skipping python prep" "debug"
return 0
fi
declare python3_binary_path="/usr/bin/python3"
# Determine what version of python3; focal-like OS's have Python 3.8, but we need 3.9.
if [[ "focal ulyana ulyssa uma una" == *"$HOSTRELEASE"* ]]; then
python3_binary_path="/usr/bin/python3.9"
display_alert "Using '${python3_binary_path}' for" "'$HOSTRELEASE' has outdated python3, using python3.9" "warn"
fi
# Check that the actual python3 --version is 3.9 at least
declare python3_version python3_full_version
python3_full_version="$("${python3_binary_path}" --version)" # "cut" below masks errors, do it twice.
python3_version="$("${python3_binary_path}" --version | cut -d' ' -f2)"
display_alert "Python3 version" "${python3_version} - '${python3_full_version}'" "info"
if ! linux-version compare "${python3_version}" ge "3.9"; then
exit_with_error "Python3 version is too old (${python3_version}), need at least 3.9"
fi
# Check actual pip3 version
# Note: we don't use "/usr/bin/pip3" at all, since it's commonly missing. instead "python -m pip"
# The hostdep package python3-pip is still required, and other crazy might impact this.
# We might need to install our own pip if it gets bad enough.
declare pip3_version
pip3_version="$("${python3_binary_path}" -m pip --version)"
# get the pip3 version number only (eg, "21.2.4" from "pip 21.2.4 from /usr/lib/python3/dist-packages/pip (python 3.9)")
declare pip3_version_number
pip3_version_number="$(echo "${pip3_version}" | cut -d' ' -f2)" # @TODO: brittle. how to do this better?
display_alert "pip3 version" "${pip3_version_number}: '${pip3_version}'" "info"
# Hash the contents of the dependencies array + the Python version + the release
declare python3_pip_dependencies_hash
early_prepare_pip3_dependencies_for_python_tools
python3_pip_dependencies_hash="$(echo "${HOSTRELEASE}" "${python3_version}" "${pip3_version}" "${python3_pip_dependencies[*]}" | sha256sum | cut -d' ' -f1)"
declare non_cache_dir="/armbian-pip"
declare python_pip_cache="${SRC}/cache/pip"
if [[ "${deploy_to_non_cache_dir:-"no"}" == "yes" ]]; then
display_alert "Using non-cache dir" "PIP: ${non_cache_dir}" "warn"
python_pip_cache="${non_cache_dir}"
else
# if the non-cache dir exists, copy it into place, if not already existing...
if [[ -d "${non_cache_dir}" && ! -d "${python_pip_cache}" ]]; then
display_alert "Deploying pip cache from Docker image" "${non_cache_dir} -> ${python_pip_cache}" "info"
run_host_command_logged cp -pr "${non_cache_dir}" "${python_pip_cache}"
fi
fi
declare -a pip3_extra_args=("--no-warn-script-location" "--user")
# if pip 23+, add "--break-system-packages" to pip3 invocations.
# See See PEP 668 -- System-wide package management with pip
# but the fact is that we're _not_ managing system-wide, instead --user
if linux-version compare "${pip3_version_number}" ge "23.0"; then
pip3_extra_args+=("--break-system-packages")
fi
if linux-version compare "${pip3_version_number}" ge "22.1"; then
pip3_extra_args+=("--root-user-action=ignore")
fi
declare python_hash_base="${python_pip_cache}/pip_pkg_hash"
declare python_hash_file="${python_hash_base}_${python3_pip_dependencies_hash}"
declare python3_user_base="${python_pip_cache}/base"
declare python3_pycache="${python_pip_cache}/pycache"
# declare a readonly global dict with all needed info for executing stuff using this setup
declare -r -g -A PYTHON3_INFO=(
[BIN]="${python3_binary_path}"
[USERBASE]="${python3_user_base}"
[PYCACHEPREFIX]="${python3_pycache}"
[HASH]="${python3_pip_dependencies_hash}"
[DEPS]="${python3_pip_dependencies[*]}"
[VERSION]="${python3_version}"
[PIP_VERSION]="${pip3_version}"
)
# declare a readonly global array for ENV vars to invoke python3 with
declare -r -g -a PYTHON3_VARS=(
"PYTHONUSERBASE=${PYTHON3_INFO[USERBASE]}"
"PYTHONUNBUFFERED=yes"
"PYTHONPYCACHEPREFIX=${PYTHON3_INFO[PYCACHEPREFIX]}"
)
# If the hash file exists, we're done.
if [[ -f "${python_hash_file}" ]]; then
display_alert "Using cached pip packages for Python tools" "${python3_pip_dependencies_hash}" "info"
else
display_alert "Installing pip packages for Python tools" "${python3_pip_dependencies_hash:0:10}" "info"
# remove the old hashes matching base, don't leave junk behind
run_host_command_logged rm -fv "${python_hash_base}*"
run_host_command_logged env -i "${PYTHON3_VARS[@]@Q}" "${PYTHON3_INFO[BIN]}" -m pip install "${pip3_extra_args[@]}" "${python3_pip_dependencies[@]}"
# Create the hash file
run_host_command_logged touch "${python_hash_file}"
fi
_already_prepared_python_and_pip="yes"
return 0
}
# Called during early_prepare_host_dependencies(); when building a Dockerfile, host_release is set to the Docker image name.
function host_deps_add_extra_python() {
# check host_release is set, or bail.
[[ -z "${host_release}" ]] && exit_with_error "host_release is not set"
# host_release is from outer scope (
# Determine what version of python3; focal-like OS's have Python 3.8, but we need 3.9.
if [[ "focal ulyana ulyssa uma una" == *"${host_release}"* ]]; then
display_alert "Using Python 3.9 for" "hostdeps: '${host_release}' has outdated python3, using python3.9" "warn"
host_dependencies+=("python3.9-dev")
else
display_alert "Using Python3 for" "hostdeps: '${host_release}' has python3 >= 3.9" "debug"
fi
}