build/lib/functions/general/python-tools.sh

#!/usr/bin/env bash
#
# SPDX-License-Identifier: GPL-2.0
#
# Copyright (c) 2013-2023 Igor Pecovnik, igor@armbian.com
#
# This file is a part of the Armbian Build Framework
# https://github.com/armbian/build/

# This whole thing is a big "I refuse to use venv in a simple bash script" delusion.
# If you know to tame it, teach me. I'd rather not know about PYTHONUSERBASE and such.
# --rpardini

function early_prepare_pip3_dependencies_for_python_tools() {
	# This is like a stupid version of requirements.txt
	declare -a -g python3_pip_dependencies=(
		"unidiff==0.7.4"      # for parsing unified diff
		"GitPython==3.1.30"   # for manipulating git repos
		"unidecode==1.3.6"    # for converting strings to ascii
		"coloredlogs==15.0.1" # for colored logging
		"PyYAML==6.0"         # for parsing/writing YAML
		"oras==0.1.17"        # for OCI stuff in mapper-oci-update
		"Jinja2==3.1.2"       # for templating
		"rich==13.4.1"        # for rich text formatting
	)
	return 0
}

# call: prepare_python_and_pip # this defines global PYTHON3_INFO dict and PYTHON3_VARS array
function prepare_python_and_pip() {
	assert_prepared_host # this needs a prepared host to work; avoid fake errors about "python3-pip" not being installed
	# First determine with python3 to use; requires knowing the HOSTRELEASE.
	[[ -z "${HOSTRELEASE}" ]] && exit_with_error "HOSTRELEASE is not set"

	# fake-memoize this, it's expensive and does not need to be done twice
	declare -g _already_prepared_python_and_pip="${_already_prepared_python_and_pip:-no}"
	if [[ "${_already_prepared_python_and_pip}" == "yes" ]]; then
		display_alert "All Python preparation done before" "skipping python prep" "debug"
		return 0
	fi

	declare python3_binary_path="/usr/bin/python3"
	# Determine what version of python3;  focal-like OS's have Python 3.8, but we need 3.9.
	if [[ "focal ulyana ulyssa uma una" == *"$HOSTRELEASE"* ]]; then
		python3_binary_path="/usr/bin/python3.9"
		display_alert "Using  '${python3_binary_path}' for" "'$HOSTRELEASE' has outdated python3, using python3.9" "warn"
	fi

	# Check that the actual python3 --version is 3.9 at least
	declare python3_version python3_full_version
	python3_full_version="$("${python3_binary_path}" --version)" # "cut" below masks errors, do it twice.
	python3_version="$("${python3_binary_path}" --version | cut -d' ' -f2)"
	display_alert "Python3 version" "${python3_version} - '${python3_full_version}'" "info"
	if ! linux-version compare "${python3_version}" ge "3.9"; then
		exit_with_error "Python3 version is too old (${python3_version}), need at least 3.9"
	fi

	# Check actual pip3 version
	#   Note: we don't use "/usr/bin/pip3" at all, since it's commonly missing. instead "python -m pip"
	#   The hostdep package python3-pip is still required, and other crazy might impact this.
	#   We might need to install our own pip if it gets bad enough.
	declare pip3_version
	pip3_version="$("${python3_binary_path}" -m pip --version)"

	# get the pip3 version number only (eg, "21.2.4" from "pip 21.2.4 from /usr/lib/python3/dist-packages/pip (python 3.9)")
	declare pip3_version_number
	pip3_version_number="$(echo "${pip3_version}" | cut -d' ' -f2)" # @TODO: brittle. how to do this better?
	display_alert "pip3 version" "${pip3_version_number}: '${pip3_version}'" "info"

	# Hash the contents of the dependencies array + the Python version + the release
	declare python3_pip_dependencies_hash
	early_prepare_pip3_dependencies_for_python_tools
	python3_pip_dependencies_hash="$(echo "${HOSTRELEASE}" "${python3_version}" "${pip3_version}" "${python3_pip_dependencies[*]}" | sha256sum | cut -d' ' -f1)"

	declare non_cache_dir="/armbian-pip"
	declare python_pip_cache="${SRC}/cache/pip"

	if [[ "${deploy_to_non_cache_dir:-"no"}" == "yes" ]]; then
		display_alert "Using non-cache dir" "PIP: ${non_cache_dir}" "warn"
		python_pip_cache="${non_cache_dir}"
	else
		# if the non-cache dir exists, copy it into place, if not already existing...
		if [[ -d "${non_cache_dir}" && ! -d "${python_pip_cache}" ]]; then
			display_alert "Deploying pip cache from Docker image" "${non_cache_dir} -> ${python_pip_cache}" "info"
			run_host_command_logged cp -pr "${non_cache_dir}" "${python_pip_cache}"
		fi
	fi

	declare -a pip3_extra_args=("--no-warn-script-location" "--user")
	# if pip 23+, add "--break-system-packages" to pip3 invocations.
	# See See PEP 668 -- System-wide package management with pip
	# but the fact is that we're _not_ managing system-wide, instead --user
	if linux-version compare "${pip3_version_number}" ge "23.0"; then
		pip3_extra_args+=("--break-system-packages")
	fi
	if linux-version compare "${pip3_version_number}" ge "22.1"; then
		pip3_extra_args+=("--root-user-action=ignore")
	fi

	declare python_hash_base="${python_pip_cache}/pip_pkg_hash"
	declare python_hash_file="${python_hash_base}_${python3_pip_dependencies_hash}"
	declare python3_user_base="${python_pip_cache}/base"
	declare python3_pycache="${python_pip_cache}/pycache"

	# declare a readonly global dict with all needed info for executing stuff using this setup
	declare -r -g -A PYTHON3_INFO=(
		[BIN]="${python3_binary_path}"
		[USERBASE]="${python3_user_base}"
		[PYCACHEPREFIX]="${python3_pycache}"
		[HASH]="${python3_pip_dependencies_hash}"
		[DEPS]="${python3_pip_dependencies[*]}"
		[VERSION]="${python3_version}"
		[PIP_VERSION]="${pip3_version}"
	)

	# declare a readonly global array for ENV vars to invoke python3 with
	declare -r -g -a PYTHON3_VARS=(
		"PYTHONUSERBASE=${PYTHON3_INFO[USERBASE]}"
		"PYTHONUNBUFFERED=yes"
		"PYTHONPYCACHEPREFIX=${PYTHON3_INFO[PYCACHEPREFIX]}"
	)

	# If the hash file exists, we're done.
	if [[ -f "${python_hash_file}" ]]; then
		display_alert "Using cached pip packages for Python tools" "${python3_pip_dependencies_hash}" "info"
	else
		display_alert "Installing pip packages for Python tools" "${python3_pip_dependencies_hash:0:10}" "info"
		# remove the old hashes matching base, don't leave junk behind
		run_host_command_logged rm -fv "${python_hash_base}*"

		run_host_command_logged env -i "${PYTHON3_VARS[@]@Q}" "${PYTHON3_INFO[BIN]}" -m pip install "${pip3_extra_args[@]}" "${python3_pip_dependencies[@]}"

		# Create the hash file
		run_host_command_logged touch "${python_hash_file}"
	fi

	_already_prepared_python_and_pip="yes"
	return 0
}

# Called during early_prepare_host_dependencies(); when building a Dockerfile, host_release is set to the Docker image name.
function host_deps_add_extra_python() {
	# check host_release is set, or bail.
	[[ -z "${host_release}" ]] && exit_with_error "host_release is not set"

	# host_release is from outer scope (
	# Determine what version of python3;  focal-like OS's have Python 3.8, but we need 3.9.
	if [[ "focal ulyana ulyssa uma una" == *"${host_release}"* ]]; then
		display_alert "Using Python 3.9 for" "hostdeps: '${host_release}' has outdated python3, using python3.9" "warn"
		host_dependencies+=("python3.9-dev")
	else
		display_alert "Using Python3 for" "hostdeps: '${host_release}' has python3 >= 3.9" "debug"
	fi
}