diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml
index 6d50ad40a10e..590f0d67b489 100644
--- a/.github/workflows/commands.yml
+++ b/.github/workflows/commands.yml
@@ -1,8 +1,12 @@
 name: commands
+
 on:
   issue_comment:
     types: [created]
 
+permissions:
+  contents: read
+
 jobs:
   retest:
     if: |
@@ -13,8 +17,10 @@ jobs:
          && github.actor != 'dependabot[bot]'
       }}
     name: Retest
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
+    permissions:
+      pull-requests: write
     steps:
-    - uses: jpsim/retest@c158dec0a7f67cb85f8367468dc8a9a75308bb7f
+    - uses: envoyproxy/toolshed/gh-actions/retest@56d5781416445ed530e075b71546dedee94cf054
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/ci/repokitteh/modules/azure_pipelines.star b/ci/repokitteh/modules/azure_pipelines.star
index 9c6cbe9ecc9f..57d6a78f0056 100644
--- a/ci/repokitteh/modules/azure_pipelines.star
+++ b/ci/repokitteh/modules/azure_pipelines.star
@@ -31,10 +31,12 @@ def _get_azp_link(check_id):
     return "https://dev.azure.com/{organization}/{project}/_build/results?buildId={buildId}&view=results".format(organization = _azp_organization, project = project, buildId = build_id)
 
 def _retry(config, comment_id, command):
-    msgs = "Retrying Azure Pipelines:\n"
+    if len(command.parts) > 1 and command.parts[1] == "mobile":
+        return
     check_ids, checks = _get_azp_checks()
 
     retried_checks = []
+    reaction = "confused"
     for check_id in check_ids:
         subchecks = [c for c in checks if c["external_id"] == check_id]
         if len(subchecks) == 0:
@@ -50,20 +52,15 @@ def _retry(config, comment_id, command):
             if check["status"] == "in_progress":
                 has_running = True
 
-        if not has_failure:
-            msgs += "Check {} didn't fail.\n".format(name_with_link)
-        else:
-            if has_running:
-                msgs += "Check {} isn't fully completed, but will still attempt retrying.\n".format(name_with_link)
+        if has_failure:
             _, build_id, project = check_id.split("|")
             _retry_azp(project, build_id, config["token"])
             retried_checks.append(name_with_link)
 
-    if len(retried_checks) == 0:
-        react(comment_id, msgs)
-    else:
-        react(comment_id, None)
-        msgs += "Retried failed jobs in: {}".format(", ".join(retried_checks))
-        github.issue_create_comment(msgs)
+    if len(retried_checks) != 0:
+        reaction = "+1"
+
+    github.issue_create_comment_reaction(comment_id, reaction)
+
 
 handlers.command(name = "retry-azp", func = _retry)