diff --git a/lib/src/main/java/org/asamk/signal/manager/ManagerImpl.java b/lib/src/main/java/org/asamk/signal/manager/ManagerImpl.java
index 5ee2e7d3..9e81f206 100644
--- a/lib/src/main/java/org/asamk/signal/manager/ManagerImpl.java
+++ b/lib/src/main/java/org/asamk/signal/manager/ManagerImpl.java
@@ -956,12 +956,12 @@ public class ManagerImpl implements Manager {
     public void removeReceiveHandler(final ReceiveMessageHandler handler) {
         final Thread thread;
         synchronized (messageHandlers) {
-            thread = receiveThread;
-            receiveThread = null;
             messageHandlers.remove(handler);
             if (!messageHandlers.isEmpty() || isReceivingSynchronous) {
                 return;
             }
+            thread = receiveThread;
+            receiveThread = null;
         }
 
         stopReceiveThread(thread);