diff --git a/src/main/java/org/asamk/signal/JsonIdentityKeyStore.java b/src/main/java/org/asamk/signal/JsonIdentityKeyStore.java
index 7cde350c..14c0d11e 100644
--- a/src/main/java/org/asamk/signal/JsonIdentityKeyStore.java
+++ b/src/main/java/org/asamk/signal/JsonIdentityKeyStore.java
@@ -58,7 +58,9 @@ class JsonIdentityKeyStore implements IdentityKeyStore {
                 if (!id.identityKey.equals(identityKey))
                     continue;
 
-                id.trustLevel = trustLevel;
+                if (id.trustLevel.compareTo(trustLevel) < 0) {
+                    id.trustLevel = trustLevel;
+                }
                 if (added != null) {
                     id.added = added;
                 }