diff --git a/.github/workflows/json-build.yml b/.github/workflows/json-build.yml
index 0a5ef4a743..96e22ad530 100644
--- a/.github/workflows/json-build.yml
+++ b/.github/workflows/json-build.yml
@@ -35,12 +35,6 @@ on:
         type: boolean
         default: true
 
-env:
-  RUSTC_WRAPPER: "/usr/bin/sccache"
-  # SCCACHE_NO_DAEMON: "1"
-  # Disable incremental compilation so sccache works better
-  CARGO_INCREMENTAL: "false"
-
 jobs:
   build:
     strategy:
@@ -75,13 +69,12 @@ jobs:
           dir=$(dirname ${{ matrix.pkg.pkg }})
           dnf5 builddep -y ${dir}/*.spec
 
-
-      - name: Run sccache-cache
-        uses: mozilla-actions/sccache-action@v0.0.9
-
       - name: Configure sccache
+        if: ${{ !contains(matrix.pkg.labels, 'sccache') }}
         run: |
           set -euo pipefail
+          echo "RUSTC_WRAPPER=/usr/bin/sccache" >> $GITHUB_ENV
+          echo "CARGO_INCREMENTAL=false" >> $GITHUB_ENV
           echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
           if [ "${{ inputs.bust_cache }}" = "true" ]; then
             echo "SCCACHE_BUST_CACHE=true" >> $GITHUB_ENV