diff --git a/readability/__init__.py b/readability/__init__.py
index 97a40cf..1593bd5 100644
--- a/readability/__init__.py
+++ b/readability/__init__.py
@@ -61,8 +61,12 @@ class Readability(Falkon.PluginInterface, QtCore.QObject):
         self.buttons[window] = button
 
     def onMainWindowDeleted(self, window):
-        if window in self.buttons:
-            del self.buttons[window]
+        if window not in self.buttons:
+            return
+
+        button = self.buttons[window]
+        window.navigationBar().removeToolButton(button)
+        del self.buttons[window]
 
     def populateWebViewMenu(self, menu, view, hitTestResult):
         if not self.config["contextMenu"]: