From e693bee89b876462092f779b9c40f56c38ccda40 Mon Sep 17 00:00:00 2001 From: madonuko Date: Thu, 24 Jul 2025 00:31:37 +0800 Subject: [PATCH] feat(ci): better build workflows --- .github/workflows/autobuild.yml | 78 ++-------------------- .github/workflows/build.yml | 108 +++++++++---------------------- .github/workflows/json-build.yml | 30 +++++++-- 3 files changed, 61 insertions(+), 155 deletions(-) diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml index 08aef1e445..1899cc7a05 100644 --- a/.github/workflows/autobuild.yml +++ b/.github/workflows/autobuild.yml @@ -38,77 +38,7 @@ jobs: run: anda ci >> $GITHUB_OUTPUT build: needs: manifest - strategy: - matrix: - pkg: ${{ fromJson(needs.manifest.outputs.build_matrix) }} - version: ["rawhide"] - fail-fast: false - runs-on: ${{ (matrix.pkg.arch == 'aarch64' && matrix.pkg.labels['large']) && 'arm64-lg' || matrix.pkg.arch == 'aarch64' && 'ubuntu-22.04-arm' || matrix.pkg.labels['large'] && 'x86-64-lg' || 'ubuntu-22.04' }} - container: - image: ghcr.io/terrapkg/builder:f${{ matrix.version }} - options: --cap-add=SYS_ADMIN --privileged - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up git repository - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: CI Setup Script - if: ${{ matrix.pkg.labels.mock != '1' }} - run: | - dir=$(dirname ${{ matrix.pkg.pkg }}) - if [ -f $dir/ci_setup.rhai ]; then - anda run $dir/ci_setup.rhai --labels script_path=$dir/ci_setup.rhai - fi - - - name: Install Build Dependencies - if: ${{ matrix.pkg.labels.mock != '1' }} - run: | - dir=$(dirname ${{ matrix.pkg.pkg }}) - dnf5 builddep -y ${dir}/*.spec - - - name: Build with Andaman - run: anda build ${{ matrix.pkg.pkg }} -D "vendor Terra" -c terra-${{ matrix.version }}-${{ matrix.pkg.arch }} ${{ !matrix.pkg.labels.mock == '1' && '-rrpmbuild' || '' }} - - - name: Generating artifact name - id: art - run: | - NAME=${{ matrix.pkg.pkg }}-${{ matrix.pkg.arch }}-${{ matrix.version }} - x=${NAME//\//@} - echo "name=$x" >> $GITHUB_OUTPUT - - - uses: actions/upload-artifact@v4 - with: - name: ${{ steps.art.outputs.name }} - compression-level: 0 # The RPMs are already compressed :p - path: | - anda-build/rpm/rpms/* - anda-build/rpm/srpm/* - - - name: Upload packages to subatomic - if: github.event_name == 'push' - run: | - subrepo="${{ matrix.pkg.labels.subrepo }}" - subatomic-cli upload --prune \ - --server https://subatomic.fyralabs.com \ - --token ${{ secrets.SUBATOMIC_TOKEN }} \ - terra${{ matrix.version }}${{ matrix.pkg.labels['subrepo'] && '-$subrepo' || '' }} anda-build/rpm/rpms/* - - - name: Upload source packages to subatomic - if: github.event_name == 'push' && matrix.pkg.labels['no_upload_srpms'] != '1' - run: | - subrepo="${{ matrix.pkg.labels.subrepo }}" - subatomic-cli upload --prune \ - --server https://subatomic.fyralabs.com \ - --token ${{ secrets.SUBATOMIC_TOKEN }} \ - terra${{ matrix.version }}${{ matrix.pkg.labels['subrepo'] && '-$subrepo' || '' }}-source anda-build/rpm/srpm/* - - - name: Notify Madoguchi (Success) - if: success() && github.event_name == 'push' - run: ./.github/workflows/mg.sh true "${{matrix.pkg.pkg}}" "${{matrix.version}}" "${{matrix.pkg.arch}}" "${{github.run_id}}" "${{secrets.MADOGUCHI_JWT}}" "$GITHUB_SHA" - - name: Notify Madoguchi (Failure) - if: ( cancelled() || failure() ) && github.event_name == 'push' - run: ./.github/workflows/mg.sh false "${{matrix.pkg.pkg}}" "${{matrix.version}}" "${{matrix.pkg.arch}}" "${{github.run_id}}" "${{secrets.MADOGUCHI_JWT}}" "$GITHUB_SHA" + uses: ./.github/workflows/json-build.yml + with: + packages: ${{ needs.manifest.outputs.build_matrix }} + publish: ${{ github.event_name == 'push' }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5982ed7085..3eeac7d780 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,95 +12,49 @@ on: required: false default: "" architecture: - description: "Architecture" + description: "Architectures" required: false - default: all + default: "" type: string jobs: - parse: + manifest: outputs: - pkgs: ${{ steps.parsing.outputs.pkgs }} - builder: ${{ inputs.custom_builder }} - arch: ${{ steps.parsing.outputs.arch }} + build_matrix: ${{ steps.parsing.outputs.build_matrix }} runs-on: ubuntu-22.04 - steps: - - name: Parse Input - id: parsing - run: | - echo "${{ inputs.packages }}" | sed 's/ /\n/g' | sed 's/$/\//g' | jq -R . | jq -s . | jq -c . | sed 's/^/pkgs=/' >> $GITHUB_OUTPUT - echo "builder=${{ inputs.custom_builder }}" >> $GITHUB_OUTPUT - arch="${{ inputs.architecture }}" - # Convert to json array using jq - # if arch is not all, convert to array - if [ "$arch" != "all" ]; then - # jq, array with single element as string - arch=$(echo $arch | sed 's/,/\n/g') - echo "arch=$(echo $arch | jq -Rs 'split("\n")' | jq 'map(select(length > 0))' | jq -c .)" >> $GITHUB_OUTPUT - else - echo "arch=$(echo '["aarch64", "x86_64"]' | jq -c .)" >> $GITHUB_OUTPUT - fi - - build: - needs: parse - strategy: - matrix: - pkg: ${{ fromJson(needs.parse.outputs.pkgs) }} - version: ["rawhide"] - arch: ${{ fromJson(needs.parse.outputs.arch) }} - fail-fast: false - runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-22.04-arm' || needs.parse.outputs.builder && needs.parse.outputs.builder || 'ubuntu-22.04' }} container: - image: ghcr.io/terrapkg/builder:f${{ matrix.version }} + image: ghcr.io/terrapkg/builder:frawhide options: --cap-add=SYS_ADMIN --privileged steps: + - name: Setup Git + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git config user.name "Raboneko" + git config user.email "raboneko@fyralabs.com" - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - - - name: Set up git repository - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Build with Andaman - run: anda build -D "vendor Terra" -c terra-${{ matrix.version }}-${{ matrix.arch }} anda/${{ matrix.pkg }}pkg - - - name: Generating artifact name - id: art + - name: Parse Input + id: parsing run: | - NAME=${{ matrix.pkg }}-${{ matrix.arch }}-${{ matrix.version }} - x=${NAME//\//@} - echo "name=$x" >> $GITHUB_OUTPUT - echo "labels=$(anda run andax/get_proj_label.rhai -l project=anda/${{ matrix.pkg }}anda.hcl)" >> $GITHUB_OUTPUT + for pkg in ${{ inputs.packages }}; do + touch anda/$pkg/.build + done + git commit -a -m "tmp" + b=$(anda ci | sed -E 's@^build_matrix=@@') + if [ "${{ inputs.architecture }}" != "" ]; then + # e.g.: [ unique_by(.pkg)[] | (.arch="x86_64", .arch="aarch64") ] + filter=`echo -n "${{ inputs.architecture }}" | tr '[:space:]' '\n' | sed -E '/^$/d; s@^.+$@.arch="\0"@' | tr '\n' ', '` + b=`echo $b | jq '[unique_by(.pkg)[] | ('"$filter"')]'` + elif [ "${{ inputs.custom_builder }}" != "" ]; then + b=`echo $b | jq 'unique_by(.pkg)'` + fi + echo "build_matrix=$b" >> $GITHUB_OUTPUT - - uses: actions/upload-artifact@v4 - with: - name: ${{ steps.art.outputs.name }} - compression-level: 0 # The RPMs are already compressed :p - path: | - anda-build/rpm/rpms/* - anda-build/rpm/srpm/* - - - name: Upload packages to subatomic - run: | - subrepo="${{ fromJson(steps.art.outputs.labels).subrepo }}" - subatomic-cli upload --prune \ - --server https://subatomic.fyralabs.com \ - --token ${{ secrets.SUBATOMIC_TOKEN }} \ - terra${{ matrix.version }}${{ fromJson(steps.art.outputs.labels)['subrepo'] && '-$subrepo' }} anda-build/rpm/rpms/* - - - name: Upload source packages to subatomic - if: fromJson(steps.art.outputs.labels)['no_upload_srpms'] != '1' - run: | - subrepo="${{ fromJson(steps.art.outputs.labels).subrepo }}" - subatomic-cli upload --prune \ - --server https://subatomic.fyralabs.com \ - --token ${{ secrets.SUBATOMIC_TOKEN }} \ - terra${{ matrix.version }}${{ fromJson(steps.art.outputs.labels)['subrepo'] && '-$subrepo' }}-source anda-build/rpm/srpm/* - - - name: Notify Madoguchi (Success) - if: success() - run: ./.github/workflows/mg.sh true "anda/${{matrix.pkg}}pkg" "${{matrix.version}}" "${{matrix.arch}}" "${{github.run_id}}" "${{secrets.MADOGUCHI_JWT}}" "$GITHUB_SHA" - - name: Notify Madoguchi (Failure) - if: cancelled() || failure() - run: ./.github/workflows/mg.sh false "anda/${{matrix.pkg}}pkg" "${{matrix.version}}" "${{matrix.arch}}" "${{github.run_id}}" "${{secrets.MADOGUCHI_JWT}}" "$GITHUB_SHA" + build: + needs: manifest + uses: ./.github/workflows/json-build.yml + with: + packages: ${{ needs.manifest.outputs.build_matrix }} + custom_builder: ${{ inputs.custom_builder }} diff --git a/.github/workflows/json-build.yml b/.github/workflows/json-build.yml index 033e16519a..a1e5350b38 100644 --- a/.github/workflows/json-build.yml +++ b/.github/workflows/json-build.yml @@ -2,11 +2,32 @@ name: JSON Build permissions: contents: read on: + workflow_call: + inputs: + packages: + description: "Packages to Build" + required: true + type: string + publish: + description: "Whether the package should be published" + required: false + type: boolean + default: true + custom_builder: + description: "Custom Builder" + required: false + default: "" workflow_dispatch: inputs: packages: description: "Packages to Build" required: true + type: string + publish: + description: "Whether the package should be published" + required: false + type: boolean + default: true jobs: build: @@ -15,7 +36,7 @@ jobs: pkg: ${{ fromJson(inputs.packages) }} version: ["rawhide"] fail-fast: false - runs-on: ${{ (matrix.pkg.arch == 'aarch64' && matrix.pkg.labels['large']) && 'arm64-lg' || matrix.pkg.arch == 'aarch64' && 'ubuntu-22.04-arm' || matrix.pkg.labels['large'] && 'x86-64-lg' || 'ubuntu-22.04' }} + runs-on: ${{ inputs.custom_builder && inputs.custom_builder || (matrix.pkg.arch == 'aarch64' && matrix.pkg.labels['large']) && 'arm64-lg' || matrix.pkg.arch == 'aarch64' && 'ubuntu-22.04-arm' || matrix.pkg.labels['large'] && 'x86-64-lg' || 'ubuntu-22.04' }} container: image: ghcr.io/terrapkg/builder:f${{ matrix.version }} options: --cap-add=SYS_ADMIN --privileged @@ -61,6 +82,7 @@ jobs: anda-build/rpm/srpm/* - name: Upload packages to subatomic + if: inputs.publish run: | subrepo="${{ matrix.pkg.labels.subrepo }}" subatomic-cli upload --prune \ @@ -69,7 +91,7 @@ jobs: terra${{ matrix.version }}${{ matrix.pkg.labels['subrepo'] && '-$subrepo' || '' }} anda-build/rpm/rpms/* - name: Upload source packages to subatomic - if: matrix.pkg.labels['no_upload_srpms'] != '1' + if: inputs.publish && matrix.pkg.labels['no_upload_srpms'] != '1' run: | subrepo="${{ matrix.pkg.labels.subrepo }}" subatomic-cli upload --prune \ @@ -78,8 +100,8 @@ jobs: terra${{ matrix.version }}${{ matrix.pkg.labels['subrepo'] && '-$subrepo' || '' }}-source anda-build/rpm/srpm/* - name: Notify Madoguchi (Success) - if: success() + if: inputs.publish && success() run: ./.github/workflows/mg.sh true "${{matrix.pkg.pkg}}" "${{matrix.version}}" "${{matrix.pkg.arch}}" "${{github.run_id}}" "${{secrets.MADOGUCHI_JWT}}" "$GITHUB_SHA" - name: Notify Madoguchi (Failure) - if: cancelled() || failure() + if: inputs.publish && (cancelled() || failure()) run: ./.github/workflows/mg.sh false "${{matrix.pkg.pkg}}" "${{matrix.version}}" "${{matrix.pkg.arch}}" "${{github.run_id}}" "${{secrets.MADOGUCHI_JWT}}" "$GITHUB_SHA"