diff --git a/test/admin_ui/test_loading_button.rb b/test/admin_ui/test_loading_button.rb
index d6e5e985a..3b8a7320d 100644
--- a/test/admin_ui/test_loading_button.rb
+++ b/test/admin_ui/test_loading_button.rb
@@ -61,6 +61,8 @@ def test_publish_button
       visit "/admin/#/config/publish"
       assert_loading_button("Publish", "Publishing...")
       assert_text("Successfully published the configuration")
+      page.execute_script("window.PNotifyRemoveAll()")
+      refute_text("Successfully published the configuration")
 
       # Verify that after the first publish, the button gets reset and can be
       # used again.
@@ -72,6 +74,8 @@ def test_publish_button
       find("nav a", :text => /Publish Changes/).click
       assert_loading_button("Publish", "Publishing...")
       assert_text("Successfully published the configuration")
+      page.execute_script("window.PNotifyRemoveAll()")
+      refute_text("Successfully published the configuration")
     end
   end