feat(ci): better build workflows [attempt 2] (#6006) (#6165)

* feat(ci): better build workflows * fix(ci/json-build): forgot about the type * chore: test if autobuild works * Revert "chore: test if autobuild works" This reverts commit 73fe8e636f. * fix(ci/build): delay setup git * fix(ci/build): need to git add all * fix(ci/build): compact output from jq * chore(ci/build): add run-name * fix(ci/autobuild): inherit secrets Signed-off-by: madomado <madonuko@outlook.com> * fix(ci/build): inherit secrets Signed-off-by: madomado <madonuko@outlook.com> --------- Signed-off-by: madomado <madonuko@outlook.com> (cherry picked from commit 4cc7cb792a) # Conflicts: # .github/workflows/autobuild.yml # .github/workflows/build.yml Co-authored-by: madomado <madonuko@outlook.com>
2026-06-05 19:32:18 +00:00 · 2025-08-26 20:04:36 -07:00
parent d05e522b26
commit 593f405e2d
3 changed files with 68 additions and 156 deletions
@@ -38,77 +38,8 @@ jobs:
        run: anda ci >> $GITHUB_OUTPUT
  build:
    needs: manifest
-    strategy:
-      matrix:
-        pkg: ${{ fromJson(needs.manifest.outputs.build_matrix) }}
-        version: ["43"]
-      fail-fast: false
-    runs-on: ${{ (matrix.pkg.arch == 'aarch64' && matrix.pkg.labels['large']) && 'arm64-lg' || matrix.pkg.arch == 'aarch64' && 'ubuntu-22.04-arm' || matrix.pkg.labels['large'] && 'cirun-x86-64-lg--${{ github.run_id }}' || 'ubuntu-22.04' }}
-    container:
-      image: ghcr.io/terrapkg/builder:f${{ matrix.version }}
-      options: --cap-add=SYS_ADMIN --privileged
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Set up git repository
-        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
-
-      - name: CI Setup Script
-        if: ${{ matrix.pkg.labels.mock != '1' }}
-        run: |
-          dir=$(dirname ${{ matrix.pkg.pkg }})
-          if [ -f $dir/ci_setup.rhai ]; then
-            anda run $dir/ci_setup.rhai --labels script_path=$dir/ci_setup.rhai
-          fi
-
-      - name: Install Build Dependencies
-        if: ${{ matrix.pkg.labels.mock != '1' }}
-        run: |
-          dir=$(dirname ${{ matrix.pkg.pkg }})
-          dnf5 builddep -y ${dir}/*.spec
-
-      - name: Build with Andaman
-        run: anda build ${{ matrix.pkg.pkg }} -D "vendor Terra" -c terra-${{ matrix.version }}-${{ matrix.pkg.arch }} ${{ !matrix.pkg.labels.mock == '1' && '-rrpmbuild' || '' }}
-
-      - name: Generating artifact name
-        id: art
-        run: |
-          NAME=${{ matrix.pkg.pkg }}-${{ matrix.pkg.arch }}-${{ matrix.version }}
-          x=${NAME//\//@}
-          echo "name=$x" >> $GITHUB_OUTPUT
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: ${{ steps.art.outputs.name }}
-          compression-level: 0 # The RPMs are already compressed :p
-          path: |
-            anda-build/rpm/rpms/*
-            anda-build/rpm/srpm/*
-
-      - name: Upload packages to subatomic
-        if: github.event_name == 'push'
-        run: |
-          subrepo="${{ matrix.pkg.labels.subrepo }}"
-          subatomic-cli upload --prune \
-            --server https://subatomic.fyralabs.com \
-            --token ${{ secrets.SUBATOMIC_TOKEN }} \
-            terra${{ matrix.version }}${{ matrix.pkg.labels['subrepo'] && '-$subrepo' || '' }} anda-build/rpm/rpms/*
-
-      - name: Upload source packages to subatomic
-        if: github.event_name == 'push' && matrix.pkg.labels['no_upload_srpms'] != '1'
-        run: |
-          subrepo="${{ matrix.pkg.labels.subrepo }}"
-          subatomic-cli upload --prune \
-            --server https://subatomic.fyralabs.com \
-            --token ${{ secrets.SUBATOMIC_TOKEN }} \
-            terra${{ matrix.version }}${{ matrix.pkg.labels['subrepo'] && '-$subrepo' || '' }}-source anda-build/rpm/srpm/*
-
-      - name: Notify Madoguchi (Success)
-        if: success() && github.event_name == 'push'
-        run: ./.github/workflows/mg.sh true "${{matrix.pkg.pkg}}" "${{matrix.version}}" "${{matrix.pkg.arch}}" "${{github.run_id}}" "${{secrets.MADOGUCHI_JWT}}" "$GITHUB_SHA"
-      - name: Notify Madoguchi (Failure)
-        if: ( cancelled() || failure() ) && github.event_name == 'push'
-        run: ./.github/workflows/mg.sh false "${{matrix.pkg.pkg}}" "${{matrix.version}}" "${{matrix.pkg.arch}}" "${{github.run_id}}" "${{secrets.MADOGUCHI_JWT}}" "$GITHUB_SHA"
+    uses: ./.github/workflows/json-build.yml
+    secrets: inherit
+    with:
+      packages: ${{ needs.manifest.outputs.build_matrix }}
+      publish: ${{ github.event_name == 'push' }}
@@ -12,95 +12,53 @@ on:
        required: false
        default: ""
      architecture:
-        description: "Architecture"
+        description: "Architectures"
        required: false
-        default: all
+        default: ""
        type: string

-jobs:
-  parse:
-    outputs:
-      pkgs: ${{ steps.parsing.outputs.pkgs }}
-      builder: ${{ inputs.custom_builder }}
-      arch: ${{ steps.parsing.outputs.arch }}
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Parse Input
-        id: parsing
-        run: |
-          echo "${{ inputs.packages }}" | sed 's/ /\n/g' | sed 's/$/\//g' | jq -R . | jq -s . | jq -c . | sed 's/^/pkgs=/' >> $GITHUB_OUTPUT
-          echo "builder=${{ inputs.custom_builder }}" >> $GITHUB_OUTPUT
-          arch="${{ inputs.architecture }}"
-          # Convert to json array using jq
-          # if arch is not all, convert to array
-          if [ "$arch" != "all" ]; then
-            # jq, array with single element as string
-             arch=$(echo $arch | sed 's/,/\n/g')
-             echo "arch=$(echo $arch | jq -Rs 'split("\n")' | jq 'map(select(length > 0))' | jq -c .)" >> $GITHUB_OUTPUT
-          else
-             echo "arch=$(echo '["aarch64", "x86_64"]' | jq -c .)" >> $GITHUB_OUTPUT
-          fi
+run-name: ${{ inputs.packages }}

-  build:
-    needs: parse
-    strategy:
-      matrix:
-        pkg: ${{ fromJson(needs.parse.outputs.pkgs) }}
-        version: ["43"]
-        arch: ${{ fromJson(needs.parse.outputs.arch) }}
-      fail-fast: false
-    runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-22.04-arm' || needs.parse.outputs.builder && needs.parse.outputs.builder || 'ubuntu-22.04' }}
+jobs:
+  manifest:
+    outputs:
+      build_matrix: ${{ steps.parsing.outputs.build_matrix }}
+    runs-on: ubuntu-22.04
    container:
-      image: ghcr.io/terrapkg/builder:f${{ matrix.version }}
+      image: ghcr.io/terrapkg/builder:frawhide
      options: --cap-add=SYS_ADMIN --privileged
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
-
-      - name: Set up git repository
-        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
-
-      - name: Build with Andaman
-        run: anda build -D "vendor Terra" -c terra-${{ matrix.version }}-${{ matrix.arch }} anda/${{ matrix.pkg }}pkg
-
-      - name: Generating artifact name
-        id: art
+      - name: Setup Git
        run: |
-          NAME=${{ matrix.pkg }}-${{ matrix.arch }}-${{ matrix.version }}
-          x=${NAME//\//@}
-          echo "name=$x" >> $GITHUB_OUTPUT
-          echo "labels=$(anda run andax/get_proj_label.rhai -l project=anda/${{ matrix.pkg }}anda.hcl)" >> $GITHUB_OUTPUT
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: ${{ steps.art.outputs.name }}
-          compression-level: 0 # The RPMs are already compressed :p
-          path: |
-            anda-build/rpm/rpms/*
-            anda-build/rpm/srpm/*
-
-      - name: Upload packages to subatomic
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          git config user.name "Raboneko"
+          git config user.email "raboneko@fyralabs.com"
+      - name: Parse Input
+        id: parsing
        run: |
-          subrepo="${{ fromJson(steps.art.outputs.labels).subrepo }}"
-          subatomic-cli upload --prune \
-            --server https://subatomic.fyralabs.com \
-            --token ${{ secrets.SUBATOMIC_TOKEN }} \
-            terra${{ matrix.version }}${{ fromJson(steps.art.outputs.labels)['subrepo'] && '-$subrepo' }} anda-build/rpm/rpms/*
+          for pkg in ${{ inputs.packages }}; do
+            touch anda/$pkg/.build
+          done
+          git add --all
+          git commit -a -m "tmp"
+          b=$(anda ci | sed -E 's@^build_matrix=@@')
+          if [ "${{ inputs.architecture }}" != "" ]; then
+            # e.g.: [ unique_by(.pkg)[] | (.arch="x86_64", .arch="aarch64") ]
+            filter=`echo -n "${{ inputs.architecture }}" | tr '[:space:]' '\n' | sed -E '/^$/d; s@^.+$@.arch="\0"@' | tr '\n' ', '`
+            b=`echo $b | jq -c '[unique_by(.pkg)[] | ('"$filter"')]'`
+          elif [ "${{ inputs.custom_builder }}" != "" ]; then
+            b=`echo $b | jq -c 'unique_by(.pkg)'`
+          fi
+          echo "build_matrix=$b" >> $GITHUB_OUTPUT

-      - name: Upload source packages to subatomic
-        if: fromJson(steps.art.outputs.labels)['no_upload_srpms'] != '1'
-        run: |
-          subrepo="${{ fromJson(steps.art.outputs.labels).subrepo }}"
-          subatomic-cli upload --prune \
-            --server https://subatomic.fyralabs.com \
-            --token ${{ secrets.SUBATOMIC_TOKEN }} \
-            terra${{ matrix.version }}${{ fromJson(steps.art.outputs.labels)['subrepo'] && '-$subrepo' }}-source anda-build/rpm/srpm/*
-
-      - name: Notify Madoguchi (Success)
-        if: success()
-        run: ./.github/workflows/mg.sh true "anda/${{matrix.pkg}}pkg" "${{matrix.version}}" "${{matrix.arch}}" "${{github.run_id}}" "${{secrets.MADOGUCHI_JWT}}" "$GITHUB_SHA"
-      - name: Notify Madoguchi (Failure)
-        if: cancelled() || failure()
-        run: ./.github/workflows/mg.sh false "anda/${{matrix.pkg}}pkg" "${{matrix.version}}" "${{matrix.arch}}" "${{github.run_id}}" "${{secrets.MADOGUCHI_JWT}}" "$GITHUB_SHA"
+  build:
+    needs: manifest
+    uses: ./.github/workflows/json-build.yml
+    secrets: inherit
+    with:
+      packages: ${{ needs.manifest.outputs.build_matrix }}
+      custom_builder: ${{ inputs.custom_builder }}
@@ -2,11 +2,33 @@ name: JSON Build
 permissions:
  contents: read
 on:
+  workflow_call:
+    inputs:
+      packages:
+        description: "Packages to Build"
+        required: true
+        type: string
+      publish:
+        description: "Whether the package should be published"
+        required: false
+        type: boolean
+        default: true
+      custom_builder:
+        description: "Custom Builder"
+        required: false
+        type: string
+        default: ""
  workflow_dispatch:
    inputs:
      packages:
        description: "Packages to Build"
        required: true
+        type: string
+      publish:
+        description: "Whether the package should be published"
+        required: false
+        type: boolean
+        default: true

 jobs:
  build:
@@ -15,7 +37,7 @@ jobs:
        pkg: ${{ fromJson(inputs.packages) }}
        version: ["43"]
      fail-fast: false
-    runs-on: ${{ (matrix.pkg.arch == 'aarch64' && matrix.pkg.labels['large']) && 'arm64-lg' || matrix.pkg.arch == 'aarch64' && 'ubuntu-22.04-arm' || matrix.pkg.labels['large'] && 'cirun-x86-64-lg--${{ github.run_id }}"' || 'ubuntu-22.04' }}
+    runs-on: ${{ inputs.custom_builder && inputs.custom_builder || (matrix.pkg.arch == 'aarch64' && matrix.pkg.labels['large']) && 'arm64-lg' || matrix.pkg.arch == 'aarch64' && 'ubuntu-22.04-arm' || matrix.pkg.labels['large'] && 'cirun-x86-64-lg--${{ github.run_id }}"' || 'ubuntu-22.04' }}
    container:
      image: ghcr.io/terrapkg/builder:f${{ matrix.version }}
      options: --cap-add=SYS_ADMIN --privileged
@@ -61,6 +83,7 @@ jobs:
            anda-build/rpm/srpm/*

      - name: Upload packages to subatomic
+        if: inputs.publish
        run: |
          subrepo="${{ matrix.pkg.labels.subrepo }}"
          subatomic-cli upload --prune \
@@ -69,7 +92,7 @@ jobs:
            terra${{ matrix.version }}${{ matrix.pkg.labels['subrepo'] && '-$subrepo' || '' }} anda-build/rpm/rpms/*

      - name: Upload source packages to subatomic
-        if: matrix.pkg.labels['no_upload_srpms'] != '1'
+        if: inputs.publish && matrix.pkg.labels['no_upload_srpms'] != '1'
        run: |
          subrepo="${{ matrix.pkg.labels.subrepo }}"
          subatomic-cli upload --prune \
@@ -78,8 +101,8 @@ jobs:
            terra${{ matrix.version }}${{ matrix.pkg.labels['subrepo'] && '-$subrepo' || '' }}-source anda-build/rpm/srpm/*

      - name: Notify Madoguchi (Success)
-        if: success()
+        if: inputs.publish && success()
        run: ./.github/workflows/mg.sh true "${{matrix.pkg.pkg}}" "${{matrix.version}}" "${{matrix.pkg.arch}}" "${{github.run_id}}" "${{secrets.MADOGUCHI_JWT}}" "$GITHUB_SHA"
      - name: Notify Madoguchi (Failure)
-        if: cancelled() || failure()
+        if: inputs.publish && (cancelled() || failure())
        run: ./.github/workflows/mg.sh false "${{matrix.pkg.pkg}}" "${{matrix.version}}" "${{matrix.pkg.arch}}" "${{github.run_id}}" "${{secrets.MADOGUCHI_JWT}}" "$GITHUB_SHA"