diff --git a/.github/workflows/bump-llamacpp.yml b/.github/workflows/bump-llamacpp.yml
index 241c5b596..5a7710c94 100644
--- a/.github/workflows/bump-llamacpp.yml
+++ b/.github/workflows/bump-llamacpp.yml
@@ -1,7 +1,8 @@
 name: Bump llama.cpp
 
+run-name: "Bump llama.cpp${{ inputs.tag && format(' to {0}', inputs.tag) || '' }}"
+
 permissions:
-  actions: write
   contents: write
   pull-requests: write
 
@@ -31,25 +32,28 @@ jobs:
           echo "$OUTPUT"
           if git diff --cached --quiet; then
             echo "changed=false" >> "$GITHUB_OUTPUT"
+            echo "### Already up to date" >> "$GITHUB_STEP_SUMMARY"
           else
             COMMIT_MSG=$(echo "$OUTPUT" | grep -oP "(?<=-m ').*(?=')")
+            TAG=$(echo "$COMMIT_MSG" | grep -oP '[^/]+(?=\)$)')
             echo "commit-msg=$COMMIT_MSG" >> "$GITHUB_OUTPUT"
+            echo "tag=$TAG" >> "$GITHUB_OUTPUT"
             echo "changed=true" >> "$GITHUB_OUTPUT"
+            echo "### $COMMIT_MSG" >> "$GITHUB_STEP_SUMMARY"
           fi
 
       - name: Create pull request
         if: steps.bump.outputs.changed == 'true'
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMIT_MSG: ${{ steps.bump.outputs.commit-msg }}
-        run: |
-          TAG=$(echo "$COMMIT_MSG" | grep -oP '[^/]+(?=\)$)')
-          gh api -X PATCH "/repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
-            -f name="Bump llama.cpp to $TAG"
-          BRANCH="bump-llamacpp-$TAG"
-          git checkout -b "$BRANCH"
-          git commit --signoff -m "$COMMIT_MSG"
-          git push -u origin "$BRANCH"
-          gh pr create \
-            --title "chore: bump llama.cpp to $TAG" \
-            --body "Bumps llama.cpp submodule to [$TAG](https://github.com/ggml-org/llama.cpp/releases/$TAG)."
+        uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          sign-commits: 'true'
+          commit-message: ${{ steps.bump.outputs.commit-msg }}
+          title: ${{ steps.bump.outputs.commit-msg }}
+          body: |
+            Bumps llama.cpp submodule.
+          branch: bump-llamacpp-${{ steps.bump.outputs.tag }}
+          base: main
+          labels: dependencies
+          delete-branch: true
+          add-paths: llamacpp/native/vendor/llama.cpp