Move sticker store to database

This commit is contained in:
AsamK 2022-06-07 15:09:10 +02:00
parent 862c2fec87
commit 9a698929f4
9 changed files with 224 additions and 145 deletions

View file

@ -32,7 +32,6 @@ import org.asamk.signal.manager.api.RecipientIdentifier;
import org.asamk.signal.manager.api.SendGroupMessageResults;
import org.asamk.signal.manager.api.SendMessageResult;
import org.asamk.signal.manager.api.SendMessageResults;
import org.asamk.signal.manager.api.StickerPack;
import org.asamk.signal.manager.api.StickerPackId;
import org.asamk.signal.manager.api.StickerPackInvalidException;
import org.asamk.signal.manager.api.StickerPackUrl;
@ -58,7 +57,7 @@ import org.asamk.signal.manager.storage.recipients.Recipient;
import org.asamk.signal.manager.storage.recipients.RecipientId;
import org.asamk.signal.manager.storage.stickerPacks.JsonStickerPack;
import org.asamk.signal.manager.storage.stickerPacks.StickerPackStore;
import org.asamk.signal.manager.storage.stickers.Sticker;
import org.asamk.signal.manager.storage.stickers.StickerPack;
import org.asamk.signal.manager.util.AttachmentUtils;
import org.asamk.signal.manager.util.KeyUtils;
import org.asamk.signal.manager.util.StickerUtils;
@ -580,7 +579,7 @@ class ManagerImpl implements Manager {
if (stickerPack == null) {
throw new InvalidStickerException("Sticker pack not found");
}
final var manifest = context.getStickerHelper().getOrRetrieveStickerPack(packId, stickerPack.getPackKey());
final var manifest = context.getStickerHelper().getOrRetrieveStickerPack(packId, stickerPack.packKey());
if (manifest.stickers().size() <= stickerId) {
throw new InvalidStickerException("Sticker id not part of this pack");
}
@ -590,7 +589,7 @@ class ManagerImpl implements Manager {
throw new InvalidStickerException("Missing local sticker file");
}
messageBuilder.withSticker(new SignalServiceDataMessage.Sticker(packId.serialize(),
stickerPack.getPackKey(),
stickerPack.packKey(),
stickerId,
manifestSticker.emoji(),
AttachmentUtils.createAttachmentStream(streamDetails, Optional.empty())));
@ -796,21 +795,21 @@ class ManagerImpl implements Manager {
var packIdString = messageSender.uploadStickerManifest(manifest, packKey);
var packId = StickerPackId.deserialize(Hex.fromStringCondensed(packIdString));
var sticker = new Sticker(packId, packKey);
account.getStickerStore().updateSticker(sticker);
var sticker = new StickerPack(packId, packKey);
account.getStickerStore().addStickerPack(sticker);
return new StickerPackUrl(packId, packKey);
}
@Override
public List<StickerPack> getStickerPacks() {
public List<org.asamk.signal.manager.api.StickerPack> getStickerPacks() {
final var stickerPackStore = context.getStickerPackStore();
return account.getStickerStore().getStickerPacks().stream().map(pack -> {
if (stickerPackStore.existsStickerPack(pack.getPackId())) {
if (stickerPackStore.existsStickerPack(pack.packId())) {
try {
final var manifest = stickerPackStore.retrieveManifest(pack.getPackId());
return new StickerPack(pack.getPackId(),
new StickerPackUrl(pack.getPackId(), pack.getPackKey()),
final var manifest = stickerPackStore.retrieveManifest(pack.packId());
return new org.asamk.signal.manager.api.StickerPack(pack.packId(),
new StickerPackUrl(pack.packId(), pack.packKey()),
pack.isInstalled(),
manifest.title(),
manifest.author(),
@ -821,7 +820,7 @@ class ManagerImpl implements Manager {
}
}
return new StickerPack(pack.getPackId(), pack.getPackKey(), pack.isInstalled());
return new org.asamk.signal.manager.api.StickerPack(pack.packId(), pack.packKey(), pack.isInstalled());
}).toList();
}

View file

@ -34,7 +34,7 @@ import org.asamk.signal.manager.storage.SignalAccount;
import org.asamk.signal.manager.storage.groups.GroupInfoV1;
import org.asamk.signal.manager.storage.recipients.Profile;
import org.asamk.signal.manager.storage.recipients.RecipientId;
import org.asamk.signal.manager.storage.stickers.Sticker;
import org.asamk.signal.manager.storage.stickers.StickerPack;
import org.asamk.signal.manager.util.KeyUtils;
import org.signal.libsignal.metadata.ProtocolInvalidKeyException;
import org.signal.libsignal.metadata.ProtocolInvalidKeyIdException;
@ -439,7 +439,8 @@ public final class IncomingMessageHandler {
var sticker = account.getStickerStore().getStickerPack(stickerPackId);
if (m.getPackKey().isPresent()) {
if (sticker == null) {
sticker = new Sticker(stickerPackId, m.getPackKey().get());
sticker = new StickerPack(-1, stickerPackId, m.getPackKey().get(), installed);
account.getStickerStore().addStickerPack(sticker);
}
if (installed) {
context.getJobExecutor()
@ -447,9 +448,8 @@ public final class IncomingMessageHandler {
}
}
if (sticker != null) {
sticker.setInstalled(installed);
account.getStickerStore().updateSticker(sticker);
if (sticker != null && sticker.isInstalled() != installed) {
account.getStickerStore().updateStickerPackInstalled(sticker.packId(), installed);
}
}
}
@ -703,8 +703,8 @@ public final class IncomingMessageHandler {
final var stickerPackId = StickerPackId.deserialize(messageSticker.getPackId());
var sticker = account.getStickerStore().getStickerPack(stickerPackId);
if (sticker == null) {
sticker = new Sticker(stickerPackId, messageSticker.getPackKey());
account.getStickerStore().updateSticker(sticker);
sticker = new StickerPack(stickerPackId, messageSticker.getPackKey());
account.getStickerStore().addStickerPack(sticker);
}
context.getJobExecutor().enqueueJob(new RetrieveStickerPackJob(stickerPackId, messageSticker.getPackKey()));
}

View file

@ -4,6 +4,7 @@ import com.zaxxer.hikari.HikariDataSource;
import org.asamk.signal.manager.storage.recipients.RecipientStore;
import org.asamk.signal.manager.storage.sendLog.MessageSendLogStore;
import org.asamk.signal.manager.storage.stickers.StickerStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -14,7 +15,7 @@ import java.sql.SQLException;
public class AccountDatabase extends Database {
private final static Logger logger = LoggerFactory.getLogger(AccountDatabase.class);
private static final long DATABASE_VERSION = 2;
private static final long DATABASE_VERSION = 3;
private AccountDatabase(final HikariDataSource dataSource) {
super(logger, DATABASE_VERSION, dataSource);
@ -28,6 +29,7 @@ public class AccountDatabase extends Database {
protected void createDatabase(final Connection connection) throws SQLException {
RecipientStore.createSql(connection);
MessageSendLogStore.createSql(connection);
StickerStore.createSql(connection);
}
@Override
@ -65,5 +67,18 @@ public class AccountDatabase extends Database {
""");
}
}
if (oldVersion < 3) {
logger.debug("Updating database: Creating sticker table");
try (final var statement = connection.createStatement()) {
statement.executeUpdate("""
CREATE TABLE sticker (
_id INTEGER PRIMARY KEY,
pack_id BLOB UNIQUE NOT NULL,
pack_key BLOB NOT NULL,
installed BOOLEAN NOT NULL DEFAULT FALSE
);
""");
}
}
}
}

View file

@ -36,6 +36,7 @@ import org.asamk.signal.manager.storage.recipients.RecipientTrustedResolver;
import org.asamk.signal.manager.storage.sendLog.MessageSendLogStore;
import org.asamk.signal.manager.storage.senderKeys.SenderKeyStore;
import org.asamk.signal.manager.storage.sessions.SessionStore;
import org.asamk.signal.manager.storage.stickers.LegacyStickerStore;
import org.asamk.signal.manager.storage.stickers.StickerStore;
import org.asamk.signal.manager.storage.threads.LegacyJsonThreadStore;
import org.asamk.signal.manager.util.IOUtils;
@ -146,7 +147,6 @@ public class SignalAccount implements Closeable {
private GroupStore.Storage groupStoreStorage;
private RecipientStore recipientStore;
private StickerStore stickerStore;
private StickerStore.Storage stickerStoreStorage;
private ConfigurationStore configurationStore;
private ConfigurationStore.Storage configurationStoreStorage;
@ -216,7 +216,6 @@ public class SignalAccount implements Closeable {
signalAccount.groupStore = new GroupStore(getGroupCachePath(dataPath, accountPath),
signalAccount.getRecipientResolver(),
signalAccount::saveGroupStore);
signalAccount.stickerStore = new StickerStore(signalAccount::saveStickerStore);
signalAccount.configurationStore = new ConfigurationStore(signalAccount::saveConfigurationStore);
signalAccount.registered = false;
@ -341,7 +340,6 @@ public class SignalAccount implements Closeable {
signalAccount.groupStore = new GroupStore(getGroupCachePath(dataPath, accountPath),
signalAccount.getRecipientResolver(),
signalAccount::saveGroupStore);
signalAccount.stickerStore = new StickerStore(signalAccount::saveStickerStore);
signalAccount.configurationStore = new ConfigurationStore(signalAccount::saveConfigurationStore);
signalAccount.getRecipientTrustedResolver()
@ -659,10 +657,10 @@ public class SignalAccount implements Closeable {
}
if (rootNode.hasNonNull("stickerStore")) {
stickerStoreStorage = jsonProcessor.convertValue(rootNode.get("stickerStore"), StickerStore.Storage.class);
stickerStore = StickerStore.fromStorage(stickerStoreStorage, this::saveStickerStore);
} else {
stickerStore = new StickerStore(this::saveStickerStore);
final var storage = jsonProcessor.convertValue(rootNode.get("stickerStore"),
LegacyStickerStore.Storage.class);
LegacyStickerStore.migrate(storage, getStickerStore());
migratedLegacyConfig = true;
}
if (rootNode.hasNonNull("configurationStore")) {
@ -853,11 +851,6 @@ public class SignalAccount implements Closeable {
return false;
}
private void saveStickerStore(StickerStore.Storage storage) {
this.stickerStoreStorage = storage;
save();
}
private void saveGroupStore(GroupStore.Storage storage) {
this.groupStoreStorage = storage;
save();
@ -910,7 +903,6 @@ public class SignalAccount implements Closeable {
profileKey == null ? null : Base64.getEncoder().encodeToString(profileKey.serialize()))
.put("registered", registered)
.putPOJO("groupStore", groupStoreStorage)
.putPOJO("stickerStore", stickerStoreStorage)
.putPOJO("configurationStore", configurationStoreStorage);
try {
try (var output = new ByteArrayOutputStream()) {
@ -1147,7 +1139,7 @@ public class SignalAccount implements Closeable {
}
public StickerStore getStickerStore() {
return stickerStore;
return getOrCreate(() -> stickerStore, () -> stickerStore = new StickerStore(getAccountDatabase()));
}
public SenderKeyStore getSenderKeyStore() {

View file

@ -0,0 +1,35 @@
package org.asamk.signal.manager.storage.stickers;
import org.asamk.signal.manager.api.StickerPackId;
import java.util.Base64;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
public class LegacyStickerStore {
public static void migrate(Storage storage, StickerStore stickerStore) {
final var packIds = new HashSet<StickerPackId>();
final var stickers = storage.stickers.stream().map(s -> {
var packId = StickerPackId.deserialize(Base64.getDecoder().decode(s.packId));
if (packIds.contains(packId)) {
// Remove legacy duplicate packIds ...
return null;
}
packIds.add(packId);
var packKey = Base64.getDecoder().decode(s.packKey);
var installed = s.installed;
return new StickerPack(-1, packId, packKey, installed);
}).filter(Objects::nonNull).toList();
stickerStore.addLegacyStickers(stickers);
}
public record Storage(List<Sticker> stickers) {
private record Sticker(String packId, String packKey, boolean installed) {
}
}
}

View file

@ -1,37 +0,0 @@
package org.asamk.signal.manager.storage.stickers;
import org.asamk.signal.manager.api.StickerPackId;
public class Sticker {
private final StickerPackId packId;
private final byte[] packKey;
private boolean installed;
public Sticker(final StickerPackId packId, final byte[] packKey) {
this.packId = packId;
this.packKey = packKey;
}
public Sticker(final StickerPackId packId, final byte[] packKey, final boolean installed) {
this.packId = packId;
this.packKey = packKey;
this.installed = installed;
}
public StickerPackId getPackId() {
return packId;
}
public byte[] getPackKey() {
return packKey;
}
public boolean isInstalled() {
return installed;
}
public void setInstalled(final boolean installed) {
this.installed = installed;
}
}

View file

@ -0,0 +1,10 @@
package org.asamk.signal.manager.storage.stickers;
import org.asamk.signal.manager.api.StickerPackId;
public record StickerPack(long internalId, StickerPackId packId, byte[] packKey, boolean isInstalled) {
public StickerPack(final StickerPackId packId, final byte[] packKey) {
this(-1, packId, packKey, false);
}
}

View file

@ -1,86 +1,149 @@
package org.asamk.signal.manager.storage.stickers;
import org.asamk.signal.manager.api.StickerPackId;
import org.asamk.signal.manager.storage.Database;
import org.asamk.signal.manager.storage.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Base64;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
public class StickerStore {
private final Map<StickerPackId, Sticker> stickers;
private final static Logger logger = LoggerFactory.getLogger(StickerStore.class);
private static final String TABLE_STICKER = "sticker";
private final Saver saver;
private final Database database;
public StickerStore(final Saver saver) {
this.saver = saver;
stickers = new HashMap<>();
public static void createSql(Connection connection) throws SQLException {
// When modifying the CREATE statement here, also add a migration in AccountDatabase.java
try (final var statement = connection.createStatement()) {
statement.executeUpdate("""
CREATE TABLE sticker (
_id INTEGER PRIMARY KEY,
pack_id BLOB UNIQUE NOT NULL,
pack_key BLOB NOT NULL,
installed BOOLEAN NOT NULL DEFAULT FALSE
);
""");
}
}
public StickerStore(final Map<StickerPackId, Sticker> stickers, final Saver saver) {
this.stickers = stickers;
this.saver = saver;
public StickerStore(final Database database) {
this.database = database;
}
public static StickerStore fromStorage(Storage storage, Saver saver) {
final var packIds = new HashSet<StickerPackId>();
final var stickers = storage.stickers.stream().map(s -> {
var packId = StickerPackId.deserialize(Base64.getDecoder().decode(s.packId));
if (packIds.contains(packId)) {
// Remove legacy duplicate packIds ...
return null;
public Collection<StickerPack> getStickerPacks() {
final var sql = (
"""
SELECT s._id, s.pack_id, s.pack_key, s.installed
FROM %s s
"""
).formatted(TABLE_STICKER);
try (final var connection = database.getConnection()) {
try (final var statement = connection.prepareStatement(sql)) {
try (var result = Utils.executeQueryForStream(statement, this::getStickerPackFromResultSet)) {
return result.toList();
}
}
packIds.add(packId);
var packKey = Base64.getDecoder().decode(s.packKey);
var installed = s.installed;
return new Sticker(packId, packKey, installed);
}).filter(Objects::nonNull).collect(Collectors.toMap(Sticker::getPackId, s -> s));
return new StickerStore(stickers, saver);
}
public Collection<Sticker> getStickerPacks() {
return stickers.values();
}
public Sticker getStickerPack(StickerPackId packId) {
synchronized (stickers) {
return stickers.get(packId);
} catch (SQLException e) {
throw new RuntimeException("Failed read from sticker store", e);
}
}
public void updateSticker(Sticker sticker) {
Storage storage;
synchronized (stickers) {
stickers.put(sticker.getPackId(), sticker);
storage = toStorageLocked();
}
saver.save(storage);
}
private Storage toStorageLocked() {
return new Storage(stickers.values()
.stream()
.map(s -> new Storage.Sticker(Base64.getEncoder().encodeToString(s.getPackId().serialize()),
Base64.getEncoder().encodeToString(s.getPackKey()),
s.isInstalled()))
.toList());
}
public record Storage(List<Storage.Sticker> stickers) {
private record Sticker(String packId, String packKey, boolean installed) {
public StickerPack getStickerPack(StickerPackId packId) {
final var sql = (
"""
SELECT s._id, s.pack_id, s.pack_key, s.installed
FROM %s s
WHERE s.pack_id = ?
"""
).formatted(TABLE_STICKER);
try (final var connection = database.getConnection()) {
try (final var statement = connection.prepareStatement(sql)) {
statement.setBytes(1, packId.serialize());
return Utils.executeQueryForOptional(statement, this::getStickerPackFromResultSet).orElse(null);
}
} catch (SQLException e) {
throw new RuntimeException("Failed read from sticker store", e);
}
}
public interface Saver {
public void addStickerPack(StickerPack stickerPack) {
final var sql = (
"""
INSERT INTO %s (pack_id, pack_key, installed)
VALUES (?, ?, ?)
"""
).formatted(TABLE_STICKER);
try (final var connection = database.getConnection()) {
try (final var statement = connection.prepareStatement(sql)) {
statement.setBytes(1, stickerPack.packId().serialize());
statement.setBytes(2, stickerPack.packKey());
statement.setBoolean(3, stickerPack.isInstalled());
statement.executeUpdate();
}
} catch (SQLException e) {
throw new RuntimeException("Failed update sticker store", e);
}
}
void save(Storage storage);
public void updateStickerPackInstalled(StickerPackId stickerPackId, boolean installed) {
final var sql = (
"""
UPDATE %s
SET installed = ?
WHERE pack_id = ?
"""
).formatted(TABLE_STICKER);
try (final var connection = database.getConnection()) {
try (final var statement = connection.prepareStatement(sql)) {
statement.setBytes(1, stickerPackId.serialize());
statement.setBoolean(2, installed);
statement.executeUpdate();
}
} catch (SQLException e) {
throw new RuntimeException("Failed update sticker store", e);
}
}
void addLegacyStickers(Collection<StickerPack> stickerPacks) {
logger.debug("Migrating legacy stickers to database");
long start = System.nanoTime();
final var sql = (
"""
INSERT INTO %s (pack_id, pack_key, installed)
VALUES (?, ?, ?)
"""
).formatted(TABLE_STICKER);
try (final var connection = database.getConnection()) {
connection.setAutoCommit(false);
try (final var statement = connection.prepareStatement("DELETE FROM %s".formatted(TABLE_STICKER))) {
statement.executeUpdate();
}
try (final var statement = connection.prepareStatement(sql)) {
for (final var sticker : stickerPacks) {
statement.setBytes(1, sticker.packId().serialize());
statement.setBytes(2, sticker.packKey());
statement.setBoolean(3, sticker.isInstalled());
statement.executeUpdate();
}
}
connection.commit();
} catch (SQLException e) {
throw new RuntimeException("Failed update sticker store", e);
}
logger.debug("Stickers migration took {}ms", (System.nanoTime() - start) / 1000000);
}
private StickerPack getStickerPackFromResultSet(ResultSet resultSet) throws SQLException {
final var internalId = resultSet.getLong("_id");
final var packId = resultSet.getBytes("pack_id");
final var packKey = resultSet.getBytes("pack_key");
final var installed = resultSet.getBoolean("installed");
return new StickerPack(internalId, StickerPackId.deserialize(packId), packKey, installed);
}
}