#!/usr/bin/env python3 # ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹ # SPDX-License-Identifier: GPL-2.0 # Copyright (c) 2023 Ricardo Pardini # This file is a part of the Armbian Build Framework https://github.com/armbian/build/ # ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹ import json import logging import os import sys sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from common import armbian_utils from common import gha # Prepare logging armbian_utils.setup_logging() log: logging.Logger = logging.getLogger("output-gha-matrix") class BuildJob(gha.BaseWorkflowJob): def __init__(self, id: str, name: str): super().__init__(id, name) self.add_default_envs() self.add_ghcr_login_step() def add_default_envs(self): self.envs["OCI_TARGET_BASE"] = "ghcr.io/${{ github.repository }}/" # This is picked up by the Docker launcher automatically self.envs["DOCKER_ARMBIAN_BASE_COORDINATE_PREFIX"] = "ghcr.io/${{ github.repository }}:armbian-next-" # Use Docker image in same repo self.envs[ "DOCKER_SKIP_UPDATE"] = "yes" # Do not apt update/install/requirements/etc during Dockerfile build, trust DOCKER_ARMBIAN_BASE_COORDINATE_PREFIX's images are up-to-date def add_ghcr_login_step(self): # Login to ghcr.io, we're gonna do a lot of OCI lookups. login_step = self.add_step("docker-login-ghcr", "Docker Login to GitHub Container Registry") login_step.uses = "docker/login-action@v2" login_step.withs["registry"] = "ghcr.io" login_step.withs["username"] = "${{ github.repository_owner }}" # GitHub username or org login_step.withs["password"] = "${{ secrets.GITHUB_TOKEN }}" # GitHub actions builtin token. repo has to have pkg access. # # Login to ghcr.io, we're gonna do a lot of OCI lookups. # - name: Docker Login to GitHub Container Registry # uses: docker/login-action@v2 # with: # registry: ghcr.io # username: ${{ github.repository_owner }} # GitHub username or org # password: ${{ secrets.GITHUB_TOKEN }} # GitHub actions builtin token. repo has to have pkg access. class ArtifactJob(BuildJob): def __init__(self, id: str, name: str): super().__init__(id, name) class ImageJob(BuildJob): def __init__(self, id: str, name: str): super().__init__(id, name) class PrepareJob(BuildJob): def __init__(self, id: str, name: str): super().__init__(id, name) def add_initial_checkout(self): # Checkout the build repo checkout_step = self.add_step("checkout-build-repo", "Checkout build repo") checkout_step.uses = "actions/checkout@v3" checkout_step.withs["repository"] = "${{ github.repository_owner }}/armbian-build" checkout_step.withs["ref"] = "extensions" checkout_step.withs["fetch-depth"] = 1 checkout_step.withs["clean"] = "false" # Now grab the SHA1 from the checked out copy grab_sha1_step = self.add_step("git-info", "Grab SHA1") grab_sha1_step.run = 'echo "sha1=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT' self.add_job_output_from_step(grab_sha1_step, "sha1") def add_cache_restore_step(self): # Restore the cache restore_cache_step = self.add_step("restore-cache", "Restore cache") restore_cache_step.uses = "actions/cache@v3" restore_cache_step.withs["path"] = "cache/memoize\ncache/oci/positive" restore_cache_step.withs["key"] = '${{ runner.os }}-cache-${{ github.sha }}-${{ steps.git-info.outputs.sha1 }}' restore_cache_step.withs["restore-keys"] = '${{ runner.os }}-matrix-cache-' def add_cache_chown_step(self): # chown the cache back to normal user chown_cache_step = self.add_step("chown-cache", "Chown cache") chown_cache_step.run = 'sudo chown -R $USER:$USER cache/memoize cache/oci/positive' def prepare_gh_releases_step(self): # @TODO this is outdated, needs replacement. Also it deletes the release if it already exists, which is not what we want. Might be necessary to move the tag. gh_releases_step = self.add_step("gh-releases", "Prepare GitHub Releases") gh_releases_step.uses = "marvinpinto/action-automatic-releases@latest" gh_releases_step.withs["repo_token"] = "${{ secrets.GITHUB_TOKEN }}" gh_releases_step.withs["automatic_release_tag"] = "latest-images" gh_releases_step.withs["prerelease"] = "false" gh_releases_step.withs["title"] = "Latest images" # read the outdated artifacts+imaes json file passed as first argument as a json object with open(sys.argv[1]) as f: info = json.load(f) # Create a WorkflowFactory wfFactory: gha.WorkflowFactory = gha.WorkflowFactory() # Create prepare job pJob: PrepareJob = PrepareJob(f"prepare", f"prepare all") pJob.set_runs_on(["self-hosted", "Linux", "matrix-prepare"]) # @TODO: de-hardcode? pJob.add_initial_checkout() pJob.add_cache_restore_step() pJobUpToDateStep = pJob.add_step(f"check-up-to-date", f"Check up to date") pJobUpToDateStep.run = f'rm -rfv output/info; bash ./compile.sh workflow rpardini-generic # DEBUG=yes' # The outputs are added later, for each artifact. pJob.add_cache_chown_step() pJob.prepare_gh_releases_step() wfFactory.add_job(pJob) all_artifact_jobs = {} u2date_artifact_outputs = {} for artifact_id in info["artifacts"]: artifact = info["artifacts"][artifact_id] skip = not not artifact["oci"]["up-to-date"] # if skip: # continue artifact_name = artifact['in']['artifact_name'] # desc = f"{artifact['out']['artifact_final_file_basename']}" desc = f"{artifact['out']['artifact_name']}" # runs_in = ["self-hosted", "Linux", 'armbian', f"artifact-{artifact_name}"] runs_on = "fast" # @TODO: externalize this logic. # rootfs's fo arm64 are built on self-hosted runners tagged with "rootfs-" if artifact_name in ["rootfs"]: rootfs_arch = artifact['in']['inputs']['ARCH'] # @TODO we should resolve arch _much_ ealier in the pipeline and make it standard if rootfs_arch in ["arm64"]: # (future: add armhf) runs_on = ["self-hosted", "Linux", f"rootfs-{rootfs_arch}"] # all kernels are built on self-hosted runners. if artifact_name in ["kernel"]: runs_on = ["self-hosted", "Linux", "alfa"] inputs = artifact['in']['original_inputs'] cmds = (["artifact"] + armbian_utils.map_to_armbian_params(inputs["vars"], True) + inputs["configs"]) invocation = " ".join(cmds) item = {"desc": desc, "runs_on": runs_on, "invocation": invocation} aJob: ArtifactJob = ArtifactJob(f"artifact-{artifact_id}", f"{desc}") aJob.set_runs_on(runs_on) build_step = aJob.add_step(f"build-artifact", f"Build artifact {desc}") build_step.run = f'echo "fake artifact: {invocation}"' # Add output to prepare job... & set the GHA output, right here. Hey us, it's us from the future. We're so smart. # write to a github actions output variable. use the filesystem. gha.set_gha_output(f"u2d-{artifact_id}", ("yes" if skip else "no")) output: gha.WorkflowJobOutput = pJob.add_job_output_from_step(pJobUpToDateStep, f"u2d-{artifact_id}") input: gha.WorkflowJobInput = aJob.add_job_input_from_needed_job_output(output) aJob.add_condition_from_input(input, "== 'no'") u2date_output: gha.WorkflowJobOutput = aJob.add_job_output_from_input(f"up-to-date-artifact", input) all_artifact_jobs[artifact_id] = aJob u2date_artifact_outputs[artifact_id] = u2date_output wfFactory.add_job(aJob) # Ok now the images... for image_id in info["images"]: image = info["images"][image_id] # skip = image["outdated_artifacts_count"] == 0 # if skip: # continue desc = f"{image['image_file_id']} {image_id}" runs_on = "fast" image_arch = image['out']['ARCH'] if image_arch in ["arm64"]: # , "armhf" runs_on = ["self-hosted", "Linux", f"image-{image_arch}"] inputs = image['in'] cmds = (armbian_utils.map_to_armbian_params(inputs["vars"], True) + inputs["configs"]) # image build is "build" command, omitted here invocation = " ".join(cmds) iJob: ImageJob = ImageJob(f"image-{image_id}", f"{desc}") iJob.set_runs_on(runs_on) build_step = iJob.add_step(f"build-image", f"Build image {desc}") build_step.run = f'echo "fake image: {invocation}"' # Make it use the outputs from the artifacts needed for this image for artifact_id in image["artifact_ids"]: log.info(f"Image {image_id} wants artifact {artifact_id}") aJob = all_artifact_jobs[artifact_id] aJobU2dOutput = u2date_artifact_outputs[artifact_id] u2dinput = iJob.add_job_input_from_needed_job_output(aJobU2dOutput) iJob.add_condition_from_input(u2dinput, "== 'no'") wfFactory.add_job(iJob) # Convert gha_workflow to YAML gha_workflow_yaml = armbian_utils.to_yaml(wfFactory.render_yaml()) # Write the YAML the target file with open(sys.argv[2], "w") as f: f.write(gha_workflow_yaml)