Move sticker store to database

graalvm-config-dir/reflect-config.json

@@ -1209,6 +1209,20 @@
     {"name":"id","parameterTypes":[] }
   ]
 },
+{
+  "name":"org.asamk.signal.manager.storage.stickers.LegacyStickerStore$Storage",
+  "allDeclaredFields":true,
+  "queryAllDeclaredMethods":true,
+  "queryAllDeclaredConstructors":true,
+  "methods":[{"name":"<init>","parameterTypes":["java.util.List"] }]
+},
+{
+  "name":"org.asamk.signal.manager.storage.stickers.LegacyStickerStore$Storage$Sticker",
+  "allDeclaredFields":true,
+  "queryAllDeclaredMethods":true,
+  "queryAllDeclaredConstructors":true,
+  "methods":[{"name":"<init>","parameterTypes":["java.lang.String","java.lang.String","boolean"] }]
+},
 {
   "name":"org.asamk.signal.manager.storage.stickers.StickerStore",
   "allDeclaredFields":true,
@@ -1216,18 +1230,6 @@
   "allDeclaredConstructors":true,
   "fields":[{"name":"stickers", "allowWrite":true}]
 },
-{
-  "name":"org.asamk.signal.manager.storage.stickers.StickerStore$Storage",
-  "allDeclaredFields":true,
-  "allDeclaredMethods":true,
-  "allDeclaredConstructors":true
-},
-{
-  "name":"org.asamk.signal.manager.storage.stickers.StickerStore$Storage$Sticker",
-  "allDeclaredFields":true,
-  "allDeclaredMethods":true,
-  "allDeclaredConstructors":true
-},
 {
   "name":"org.asamk.signal.util.SecurityProvider$DefaultRandom",
   "methods":[{"name":"<init>","parameterTypes":[] }]

lib/src/main/java/org/asamk/signal/manager/ManagerImpl.java

@@ -32,7 +32,6 @@ import org.asamk.signal.manager.api.RecipientIdentifier;
 import org.asamk.signal.manager.api.SendGroupMessageResults;
 import org.asamk.signal.manager.api.SendMessageResult;
 import org.asamk.signal.manager.api.SendMessageResults;
-import org.asamk.signal.manager.api.StickerPack;
 import org.asamk.signal.manager.api.StickerPackId;
 import org.asamk.signal.manager.api.StickerPackInvalidException;
 import org.asamk.signal.manager.api.StickerPackUrl;
@@ -58,7 +57,7 @@ import org.asamk.signal.manager.storage.recipients.Recipient;
 import org.asamk.signal.manager.storage.recipients.RecipientId;
 import org.asamk.signal.manager.storage.stickerPacks.JsonStickerPack;
 import org.asamk.signal.manager.storage.stickerPacks.StickerPackStore;
-import org.asamk.signal.manager.storage.stickers.Sticker;
+import org.asamk.signal.manager.storage.stickers.StickerPack;
 import org.asamk.signal.manager.util.AttachmentUtils;
 import org.asamk.signal.manager.util.KeyUtils;
 import org.asamk.signal.manager.util.StickerUtils;
@@ -580,7 +579,7 @@ class ManagerImpl implements Manager {
             if (stickerPack == null) {
                 throw new InvalidStickerException("Sticker pack not found");
             }
-            final var manifest = context.getStickerHelper().getOrRetrieveStickerPack(packId, stickerPack.getPackKey());
+            final var manifest = context.getStickerHelper().getOrRetrieveStickerPack(packId, stickerPack.packKey());
             if (manifest.stickers().size() <= stickerId) {
                 throw new InvalidStickerException("Sticker id not part of this pack");
             }
@@ -590,7 +589,7 @@ class ManagerImpl implements Manager {
                 throw new InvalidStickerException("Missing local sticker file");
             }
             messageBuilder.withSticker(new SignalServiceDataMessage.Sticker(packId.serialize(),
-                    stickerPack.getPackKey(),
+                    stickerPack.packKey(),
                     stickerId,
                     manifestSticker.emoji(),
                     AttachmentUtils.createAttachmentStream(streamDetails, Optional.empty())));
@@ -796,21 +795,21 @@ class ManagerImpl implements Manager {
         var packIdString = messageSender.uploadStickerManifest(manifest, packKey);
         var packId = StickerPackId.deserialize(Hex.fromStringCondensed(packIdString));
 
-        var sticker = new Sticker(packId, packKey);
+        var sticker = new StickerPack(packId, packKey);
-        account.getStickerStore().updateSticker(sticker);
+        account.getStickerStore().addStickerPack(sticker);
 
         return new StickerPackUrl(packId, packKey);
     }
 
     @Override
-    public List<StickerPack> getStickerPacks() {
+    public List<org.asamk.signal.manager.api.StickerPack> getStickerPacks() {
         final var stickerPackStore = context.getStickerPackStore();
         return account.getStickerStore().getStickerPacks().stream().map(pack -> {
-            if (stickerPackStore.existsStickerPack(pack.getPackId())) {
+            if (stickerPackStore.existsStickerPack(pack.packId())) {
                 try {
-                    final var manifest = stickerPackStore.retrieveManifest(pack.getPackId());
+                    final var manifest = stickerPackStore.retrieveManifest(pack.packId());
-                    return new StickerPack(pack.getPackId(),
+                    return new org.asamk.signal.manager.api.StickerPack(pack.packId(),
-                            new StickerPackUrl(pack.getPackId(), pack.getPackKey()),
+                            new StickerPackUrl(pack.packId(), pack.packKey()),
                             pack.isInstalled(),
                             manifest.title(),
                             manifest.author(),
@@ -821,7 +820,7 @@ class ManagerImpl implements Manager {
                 }
             }
 
-            return new StickerPack(pack.getPackId(), pack.getPackKey(), pack.isInstalled());
+            return new org.asamk.signal.manager.api.StickerPack(pack.packId(), pack.packKey(), pack.isInstalled());
         }).toList();
     }
 

lib/src/main/java/org/asamk/signal/manager/helper/IncomingMessageHandler.java

@@ -34,7 +34,7 @@ import org.asamk.signal.manager.storage.SignalAccount;
 import org.asamk.signal.manager.storage.groups.GroupInfoV1;
 import org.asamk.signal.manager.storage.recipients.Profile;
 import org.asamk.signal.manager.storage.recipients.RecipientId;
-import org.asamk.signal.manager.storage.stickers.Sticker;
+import org.asamk.signal.manager.storage.stickers.StickerPack;
 import org.asamk.signal.manager.util.KeyUtils;
 import org.signal.libsignal.metadata.ProtocolInvalidKeyException;
 import org.signal.libsignal.metadata.ProtocolInvalidKeyIdException;
@@ -439,7 +439,8 @@ public final class IncomingMessageHandler {
                 var sticker = account.getStickerStore().getStickerPack(stickerPackId);
                 if (m.getPackKey().isPresent()) {
                     if (sticker == null) {
-                        sticker = new Sticker(stickerPackId, m.getPackKey().get());
+                        sticker = new StickerPack(-1, stickerPackId, m.getPackKey().get(), installed);
+                        account.getStickerStore().addStickerPack(sticker);
                     }
                     if (installed) {
                         context.getJobExecutor()
@@ -447,9 +448,8 @@ public final class IncomingMessageHandler {
                     }
                 }
 
-                if (sticker != null) {
+                if (sticker != null && sticker.isInstalled() != installed) {
-                    sticker.setInstalled(installed);
+                    account.getStickerStore().updateStickerPackInstalled(sticker.packId(), installed);
-                    account.getStickerStore().updateSticker(sticker);
                 }
             }
         }
@@ -703,8 +703,8 @@ public final class IncomingMessageHandler {
             final var stickerPackId = StickerPackId.deserialize(messageSticker.getPackId());
             var sticker = account.getStickerStore().getStickerPack(stickerPackId);
             if (sticker == null) {
-                sticker = new Sticker(stickerPackId, messageSticker.getPackKey());
+                sticker = new StickerPack(stickerPackId, messageSticker.getPackKey());
-                account.getStickerStore().updateSticker(sticker);
+                account.getStickerStore().addStickerPack(sticker);
             }
             context.getJobExecutor().enqueueJob(new RetrieveStickerPackJob(stickerPackId, messageSticker.getPackKey()));
         }

lib/src/main/java/org/asamk/signal/manager/storage/AccountDatabase.java

@@ -4,6 +4,7 @@ import com.zaxxer.hikari.HikariDataSource;
 
 import org.asamk.signal.manager.storage.recipients.RecipientStore;
 import org.asamk.signal.manager.storage.sendLog.MessageSendLogStore;
+import org.asamk.signal.manager.storage.stickers.StickerStore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -14,7 +15,7 @@ import java.sql.SQLException;
 public class AccountDatabase extends Database {
 
     private final static Logger logger = LoggerFactory.getLogger(AccountDatabase.class);
-    private static final long DATABASE_VERSION = 2;
+    private static final long DATABASE_VERSION = 3;
 
     private AccountDatabase(final HikariDataSource dataSource) {
         super(logger, DATABASE_VERSION, dataSource);
@@ -28,6 +29,7 @@ public class AccountDatabase extends Database {
     protected void createDatabase(final Connection connection) throws SQLException {
         RecipientStore.createSql(connection);
         MessageSendLogStore.createSql(connection);
+        StickerStore.createSql(connection);
     }
 
     @Override
@@ -65,5 +67,18 @@ public class AccountDatabase extends Database {
                                         """);
             }
         }
+        if (oldVersion < 3) {
+            logger.debug("Updating database: Creating sticker table");
+            try (final var statement = connection.createStatement()) {
+                statement.executeUpdate("""
+                                        CREATE TABLE sticker (
+                                          _id INTEGER PRIMARY KEY,
+                                          pack_id BLOB UNIQUE NOT NULL,
+                                          pack_key BLOB NOT NULL,
+                                          installed BOOLEAN NOT NULL DEFAULT FALSE
+                                        );
+                                        """);
+            }
+        }
     }
 }

lib/src/main/java/org/asamk/signal/manager/storage/SignalAccount.java

@@ -36,6 +36,7 @@ import org.asamk.signal.manager.storage.recipients.RecipientTrustedResolver;
 import org.asamk.signal.manager.storage.sendLog.MessageSendLogStore;
 import org.asamk.signal.manager.storage.senderKeys.SenderKeyStore;
 import org.asamk.signal.manager.storage.sessions.SessionStore;
+import org.asamk.signal.manager.storage.stickers.LegacyStickerStore;
 import org.asamk.signal.manager.storage.stickers.StickerStore;
 import org.asamk.signal.manager.storage.threads.LegacyJsonThreadStore;
 import org.asamk.signal.manager.util.IOUtils;
@@ -146,7 +147,6 @@ public class SignalAccount implements Closeable {
     private GroupStore.Storage groupStoreStorage;
     private RecipientStore recipientStore;
     private StickerStore stickerStore;
-    private StickerStore.Storage stickerStoreStorage;
     private ConfigurationStore configurationStore;
     private ConfigurationStore.Storage configurationStoreStorage;
 
@@ -216,7 +216,6 @@ public class SignalAccount implements Closeable {
         signalAccount.groupStore = new GroupStore(getGroupCachePath(dataPath, accountPath),
                 signalAccount.getRecipientResolver(),
                 signalAccount::saveGroupStore);
-        signalAccount.stickerStore = new StickerStore(signalAccount::saveStickerStore);
         signalAccount.configurationStore = new ConfigurationStore(signalAccount::saveConfigurationStore);
 
         signalAccount.registered = false;
@@ -341,7 +340,6 @@ public class SignalAccount implements Closeable {
         signalAccount.groupStore = new GroupStore(getGroupCachePath(dataPath, accountPath),
                 signalAccount.getRecipientResolver(),
                 signalAccount::saveGroupStore);
-        signalAccount.stickerStore = new StickerStore(signalAccount::saveStickerStore);
         signalAccount.configurationStore = new ConfigurationStore(signalAccount::saveConfigurationStore);
 
         signalAccount.getRecipientTrustedResolver()
@@ -659,10 +657,10 @@ public class SignalAccount implements Closeable {
         }
 
         if (rootNode.hasNonNull("stickerStore")) {
-            stickerStoreStorage = jsonProcessor.convertValue(rootNode.get("stickerStore"), StickerStore.Storage.class);
+            final var storage = jsonProcessor.convertValue(rootNode.get("stickerStore"),
-            stickerStore = StickerStore.fromStorage(stickerStoreStorage, this::saveStickerStore);
+                    LegacyStickerStore.Storage.class);
-        } else {
+            LegacyStickerStore.migrate(storage, getStickerStore());
-            stickerStore = new StickerStore(this::saveStickerStore);
+            migratedLegacyConfig = true;
         }
 
         if (rootNode.hasNonNull("configurationStore")) {
@@ -853,11 +851,6 @@ public class SignalAccount implements Closeable {
         return false;
     }
 
-    private void saveStickerStore(StickerStore.Storage storage) {
-        this.stickerStoreStorage = storage;
-        save();
-    }
-
     private void saveGroupStore(GroupStore.Storage storage) {
         this.groupStoreStorage = storage;
         save();
@@ -910,7 +903,6 @@ public class SignalAccount implements Closeable {
                             profileKey == null ? null : Base64.getEncoder().encodeToString(profileKey.serialize()))
                     .put("registered", registered)
                     .putPOJO("groupStore", groupStoreStorage)
-                    .putPOJO("stickerStore", stickerStoreStorage)
                     .putPOJO("configurationStore", configurationStoreStorage);
             try {
                 try (var output = new ByteArrayOutputStream()) {
@@ -1147,7 +1139,7 @@ public class SignalAccount implements Closeable {
     }
 
     public StickerStore getStickerStore() {
-        return stickerStore;
+        return getOrCreate(() -> stickerStore, () -> stickerStore = new StickerStore(getAccountDatabase()));
     }
 
     public SenderKeyStore getSenderKeyStore() {

lib/src/main/java/org/asamk/signal/manager/storage/stickers/LegacyStickerStore.java

@@ -0,0 +1,35 @@
+package org.asamk.signal.manager.storage.stickers;
+
+import org.asamk.signal.manager.api.StickerPackId;
+
+import java.util.Base64;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+
+public class LegacyStickerStore {
+
+    public static void migrate(Storage storage, StickerStore stickerStore) {
+        final var packIds = new HashSet<StickerPackId>();
+        final var stickers = storage.stickers.stream().map(s -> {
+            var packId = StickerPackId.deserialize(Base64.getDecoder().decode(s.packId));
+            if (packIds.contains(packId)) {
+                // Remove legacy duplicate packIds ...
+                return null;
+            }
+            packIds.add(packId);
+            var packKey = Base64.getDecoder().decode(s.packKey);
+            var installed = s.installed;
+            return new StickerPack(-1, packId, packKey, installed);
+        }).filter(Objects::nonNull).toList();
+
+        stickerStore.addLegacyStickers(stickers);
+    }
+
+    public record Storage(List<Sticker> stickers) {
+
+        private record Sticker(String packId, String packKey, boolean installed) {
+
+        }
+    }
+}

lib/src/main/java/org/asamk/signal/manager/storage/stickers/Sticker.java

@@ -1,37 +0,0 @@
-package org.asamk.signal.manager.storage.stickers;
-
-import org.asamk.signal.manager.api.StickerPackId;
-
-public class Sticker {
-
-    private final StickerPackId packId;
-    private final byte[] packKey;
-    private boolean installed;
-
-    public Sticker(final StickerPackId packId, final byte[] packKey) {
-        this.packId = packId;
-        this.packKey = packKey;
-    }
-
-    public Sticker(final StickerPackId packId, final byte[] packKey, final boolean installed) {
-        this.packId = packId;
-        this.packKey = packKey;
-        this.installed = installed;
-    }
-
-    public StickerPackId getPackId() {
-        return packId;
-    }
-
-    public byte[] getPackKey() {
-        return packKey;
-    }
-
-    public boolean isInstalled() {
-        return installed;
-    }
-
-    public void setInstalled(final boolean installed) {
-        this.installed = installed;
-    }
-}

lib/src/main/java/org/asamk/signal/manager/storage/stickers/StickerPack.java

@@ -0,0 +1,10 @@
+package org.asamk.signal.manager.storage.stickers;
+
+import org.asamk.signal.manager.api.StickerPackId;
+
+public record StickerPack(long internalId, StickerPackId packId, byte[] packKey, boolean isInstalled) {
+
+    public StickerPack(final StickerPackId packId, final byte[] packKey) {
+        this(-1, packId, packKey, false);
+    }
+}

lib/src/main/java/org/asamk/signal/manager/storage/stickers/StickerStore.java

@@ -1,86 +1,149 @@
 package org.asamk.signal.manager.storage.stickers;
 
 import org.asamk.signal.manager.api.StickerPackId;
+import org.asamk.signal.manager.storage.Database;
+import org.asamk.signal.manager.storage.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import java.util.Base64;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
 
 public class StickerStore {
 
-    private final Map<StickerPackId, Sticker> stickers;
+    private final static Logger logger = LoggerFactory.getLogger(StickerStore.class);
+    private static final String TABLE_STICKER = "sticker";
 
-    private final Saver saver;
+    private final Database database;
 
-    public StickerStore(final Saver saver) {
+    public static void createSql(Connection connection) throws SQLException {
-        this.saver = saver;
+        // When modifying the CREATE statement here, also add a migration in AccountDatabase.java
-        stickers = new HashMap<>();
+        try (final var statement = connection.createStatement()) {
-    }
+            statement.executeUpdate("""
-
+                                    CREATE TABLE sticker (
-    public StickerStore(final Map<StickerPackId, Sticker> stickers, final Saver saver) {
+                                      _id INTEGER PRIMARY KEY,
-        this.stickers = stickers;
+                                      pack_id BLOB UNIQUE NOT NULL,
-        this.saver = saver;
+                                      pack_key BLOB NOT NULL,
-    }
+                                      installed BOOLEAN NOT NULL DEFAULT FALSE
-
+                                    );
-    public static StickerStore fromStorage(Storage storage, Saver saver) {
+                                    """);
-        final var packIds = new HashSet<StickerPackId>();
-        final var stickers = storage.stickers.stream().map(s -> {
-            var packId = StickerPackId.deserialize(Base64.getDecoder().decode(s.packId));
-            if (packIds.contains(packId)) {
-                // Remove legacy duplicate packIds ...
-                return null;
-            }
-            packIds.add(packId);
-            var packKey = Base64.getDecoder().decode(s.packKey);
-            var installed = s.installed;
-            return new Sticker(packId, packKey, installed);
-        }).filter(Objects::nonNull).collect(Collectors.toMap(Sticker::getPackId, s -> s));
-
-        return new StickerStore(stickers, saver);
-    }
-
-    public Collection<Sticker> getStickerPacks() {
-        return stickers.values();
-    }
-
-    public Sticker getStickerPack(StickerPackId packId) {
-        synchronized (stickers) {
-            return stickers.get(packId);
         }
     }
 
-    public void updateSticker(Sticker sticker) {
+    public StickerStore(final Database database) {
-        Storage storage;
+        this.database = database;
-        synchronized (stickers) {
-            stickers.put(sticker.getPackId(), sticker);
-            storage = toStorageLocked();
-        }
-        saver.save(storage);
     }
 
-    private Storage toStorageLocked() {
+    public Collection<StickerPack> getStickerPacks() {
-        return new Storage(stickers.values()
+        final var sql = (
-                .stream()
+                """
-                .map(s -> new Storage.Sticker(Base64.getEncoder().encodeToString(s.getPackId().serialize()),
+                SELECT s._id, s.pack_id, s.pack_key, s.installed
-                        Base64.getEncoder().encodeToString(s.getPackKey()),
+                FROM %s s
-                        s.isInstalled()))
+                """
-                .toList());
+        ).formatted(TABLE_STICKER);
+        try (final var connection = database.getConnection()) {
+            try (final var statement = connection.prepareStatement(sql)) {
+                try (var result = Utils.executeQueryForStream(statement, this::getStickerPackFromResultSet)) {
+                    return result.toList();
                 }
-
+            }
-    public record Storage(List<Storage.Sticker> stickers) {
+        } catch (SQLException e) {
-
+            throw new RuntimeException("Failed read from sticker store", e);
-        private record Sticker(String packId, String packKey, boolean installed) {
-
         }
     }
 
-    public interface Saver {
+    public StickerPack getStickerPack(StickerPackId packId) {
+        final var sql = (
+                """
+                SELECT s._id, s.pack_id, s.pack_key, s.installed
+                FROM %s s
+                WHERE s.pack_id = ?
+                """
+        ).formatted(TABLE_STICKER);
+        try (final var connection = database.getConnection()) {
+            try (final var statement = connection.prepareStatement(sql)) {
+                statement.setBytes(1, packId.serialize());
+                return Utils.executeQueryForOptional(statement, this::getStickerPackFromResultSet).orElse(null);
+            }
+        } catch (SQLException e) {
+            throw new RuntimeException("Failed read from sticker store", e);
+        }
+    }
 
-        void save(Storage storage);
+    public void addStickerPack(StickerPack stickerPack) {
+        final var sql = (
+                """
+                INSERT INTO %s (pack_id, pack_key, installed)
+                VALUES (?, ?, ?)
+                """
+        ).formatted(TABLE_STICKER);
+        try (final var connection = database.getConnection()) {
+            try (final var statement = connection.prepareStatement(sql)) {
+                statement.setBytes(1, stickerPack.packId().serialize());
+                statement.setBytes(2, stickerPack.packKey());
+                statement.setBoolean(3, stickerPack.isInstalled());
+                statement.executeUpdate();
+            }
+        } catch (SQLException e) {
+            throw new RuntimeException("Failed update sticker store", e);
+        }
+    }
+
+    public void updateStickerPackInstalled(StickerPackId stickerPackId, boolean installed) {
+        final var sql = (
+                """
+                UPDATE %s
+                SET installed = ?
+                WHERE pack_id = ?
+                """
+        ).formatted(TABLE_STICKER);
+        try (final var connection = database.getConnection()) {
+            try (final var statement = connection.prepareStatement(sql)) {
+                statement.setBytes(1, stickerPackId.serialize());
+                statement.setBoolean(2, installed);
+                statement.executeUpdate();
+            }
+        } catch (SQLException e) {
+            throw new RuntimeException("Failed update sticker store", e);
+        }
+    }
+
+    void addLegacyStickers(Collection<StickerPack> stickerPacks) {
+        logger.debug("Migrating legacy stickers to database");
+        long start = System.nanoTime();
+        final var sql = (
+                """
+                INSERT INTO %s (pack_id, pack_key, installed)
+                VALUES (?, ?, ?)
+                """
+        ).formatted(TABLE_STICKER);
+        try (final var connection = database.getConnection()) {
+            connection.setAutoCommit(false);
+            try (final var statement = connection.prepareStatement("DELETE FROM %s".formatted(TABLE_STICKER))) {
+                statement.executeUpdate();
+            }
+            try (final var statement = connection.prepareStatement(sql)) {
+                for (final var sticker : stickerPacks) {
+                    statement.setBytes(1, sticker.packId().serialize());
+                    statement.setBytes(2, sticker.packKey());
+                    statement.setBoolean(3, sticker.isInstalled());
+                    statement.executeUpdate();
+                }
+            }
+            connection.commit();
+        } catch (SQLException e) {
+            throw new RuntimeException("Failed update sticker store", e);
+        }
+        logger.debug("Stickers migration took {}ms", (System.nanoTime() - start) / 1000000);
+    }
+
+    private StickerPack getStickerPackFromResultSet(ResultSet resultSet) throws SQLException {
+        final var internalId = resultSet.getLong("_id");
+        final var packId = resultSet.getBytes("pack_id");
+        final var packKey = resultSet.getBytes("pack_key");
+        final var installed = resultSet.getBoolean("installed");
+        return new StickerPack(internalId, StickerPackId.deserialize(packId), packKey, installed);
     }
 }