use of org.thoughtcrime.securesms.database.UnknownStorageIdDatabase in project Signal-Android by WhisperSystems.
the class StorageSyncJob method performSync.
private boolean performSync() throws IOException, RetryLaterException, InvalidKeyException {
final Stopwatch stopwatch = new Stopwatch("StorageSync");
final SQLiteDatabase db = SignalDatabase.getRawDatabase();
final SignalServiceAccountManager accountManager = ApplicationDependencies.getSignalServiceAccountManager();
final UnknownStorageIdDatabase storageIdDatabase = SignalDatabase.unknownStorageIds();
final StorageKey storageServiceKey = SignalStore.storageService().getOrCreateStorageKey();
final SignalStorageManifest localManifest = SignalStore.storageService().getManifest();
final SignalStorageManifest remoteManifest = accountManager.getStorageManifestIfDifferentVersion(storageServiceKey, localManifest.getVersion()).or(localManifest);
stopwatch.split("remote-manifest");
Recipient self = freshSelf();
boolean needsMultiDeviceSync = false;
boolean needsForcePush = false;
if (self.getStorageServiceId() == null) {
Log.w(TAG, "No storageId for self. Generating.");
SignalDatabase.recipients().updateStorageId(self.getId(), StorageSyncHelper.generateKey());
self = freshSelf();
}
Log.i(TAG, "Our version: " + localManifest.getVersion() + ", their version: " + remoteManifest.getVersion());
if (remoteManifest.getVersion() > localManifest.getVersion()) {
Log.i(TAG, "[Remote Sync] Newer manifest version found!");
List<StorageId> localStorageIdsBeforeMerge = getAllLocalStorageIds(context, self);
IdDifferenceResult idDifference = StorageSyncHelper.findIdDifference(remoteManifest.getStorageIds(), localStorageIdsBeforeMerge);
if (idDifference.hasTypeMismatches() && SignalStore.account().isPrimaryDevice()) {
Log.w(TAG, "[Remote Sync] Found type mismatches in the ID sets! Scheduling a force push after this sync completes.");
needsForcePush = true;
}
Log.i(TAG, "[Remote Sync] Pre-Merge ID Difference :: " + idDifference);
stopwatch.split("remote-id-diff");
if (!idDifference.isEmpty()) {
Log.i(TAG, "[Remote Sync] Retrieving records for key difference.");
List<SignalStorageRecord> remoteOnly = accountManager.readStorageRecords(storageServiceKey, idDifference.getRemoteOnlyIds());
stopwatch.split("remote-records");
if (remoteOnly.size() != idDifference.getRemoteOnlyIds().size()) {
Log.w(TAG, "[Remote Sync] Could not find all remote-only records! Requested: " + idDifference.getRemoteOnlyIds().size() + ", Found: " + remoteOnly.size() + ". These stragglers should naturally get deleted during the sync.");
}
List<SignalContactRecord> remoteContacts = new LinkedList<>();
List<SignalGroupV1Record> remoteGv1 = new LinkedList<>();
List<SignalGroupV2Record> remoteGv2 = new LinkedList<>();
List<SignalAccountRecord> remoteAccount = new LinkedList<>();
List<SignalStorageRecord> remoteUnknown = new LinkedList<>();
for (SignalStorageRecord remote : remoteOnly) {
if (remote.getContact().isPresent()) {
remoteContacts.add(remote.getContact().get());
} else if (remote.getGroupV1().isPresent()) {
remoteGv1.add(remote.getGroupV1().get());
} else if (remote.getGroupV2().isPresent()) {
remoteGv2.add(remote.getGroupV2().get());
} else if (remote.getAccount().isPresent()) {
remoteAccount.add(remote.getAccount().get());
} else if (remote.getId().isUnknown()) {
remoteUnknown.add(remote);
} else {
Log.w(TAG, "Bad record! Type is a known value (" + remote.getId().getType() + "), but doesn't have a matching inner record. Dropping it.");
}
}
db.beginTransaction();
try {
self = freshSelf();
Log.i(TAG, "[Remote Sync] Remote-Only :: Contacts: " + remoteContacts.size() + ", GV1: " + remoteGv1.size() + ", GV2: " + remoteGv2.size() + ", Account: " + remoteAccount.size());
new ContactRecordProcessor(context, self).process(remoteContacts, StorageSyncHelper.KEY_GENERATOR);
new GroupV1RecordProcessor(context).process(remoteGv1, StorageSyncHelper.KEY_GENERATOR);
new GroupV2RecordProcessor(context).process(remoteGv2, StorageSyncHelper.KEY_GENERATOR);
self = freshSelf();
new AccountRecordProcessor(context, self).process(remoteAccount, StorageSyncHelper.KEY_GENERATOR);
List<SignalStorageRecord> unknownInserts = remoteUnknown;
List<StorageId> unknownDeletes = Stream.of(idDifference.getLocalOnlyIds()).filter(StorageId::isUnknown).toList();
Log.i(TAG, "[Remote Sync] Unknowns :: " + unknownInserts.size() + " inserts, " + unknownDeletes.size() + " deletes");
storageIdDatabase.insert(unknownInserts);
storageIdDatabase.delete(unknownDeletes);
db.setTransactionSuccessful();
} finally {
db.endTransaction();
ApplicationDependencies.getDatabaseObserver().notifyConversationListListeners();
stopwatch.split("remote-merge-transaction");
}
} else {
Log.i(TAG, "[Remote Sync] Remote version was newer, but there were no remote-only IDs.");
}
} else if (remoteManifest.getVersion() < localManifest.getVersion()) {
Log.w(TAG, "[Remote Sync] Remote version was older. User might have switched accounts.");
}
if (remoteManifest != localManifest) {
Log.i(TAG, "[Remote Sync] Saved new manifest. Now at version: " + remoteManifest.getVersion());
SignalStore.storageService().setManifest(remoteManifest);
}
Log.i(TAG, "We are up-to-date with the remote storage state.");
final WriteOperationResult remoteWriteOperation;
db.beginTransaction();
try {
self = freshSelf();
List<StorageId> localStorageIds = getAllLocalStorageIds(context, self);
IdDifferenceResult idDifference = StorageSyncHelper.findIdDifference(remoteManifest.getStorageIds(), localStorageIds);
List<SignalStorageRecord> remoteInserts = buildLocalStorageRecords(context, self, idDifference.getLocalOnlyIds());
List<byte[]> remoteDeletes = Stream.of(idDifference.getRemoteOnlyIds()).map(StorageId::getRaw).toList();
Log.i(TAG, "ID Difference :: " + idDifference);
remoteWriteOperation = new WriteOperationResult(new SignalStorageManifest(remoteManifest.getVersion() + 1, localStorageIds), remoteInserts, remoteDeletes);
db.setTransactionSuccessful();
} finally {
db.endTransaction();
stopwatch.split("local-data-transaction");
}
if (!remoteWriteOperation.isEmpty()) {
Log.i(TAG, "We have something to write remotely.");
Log.i(TAG, "WriteOperationResult :: " + remoteWriteOperation);
StorageSyncValidations.validate(remoteWriteOperation, remoteManifest, needsForcePush, self);
Optional<SignalStorageManifest> conflict = accountManager.writeStorageRecords(storageServiceKey, remoteWriteOperation.getManifest(), remoteWriteOperation.getInserts(), remoteWriteOperation.getDeletes());
if (conflict.isPresent()) {
Log.w(TAG, "Hit a conflict when trying to resolve the conflict! Retrying.");
throw new RetryLaterException();
}
Log.i(TAG, "Saved new manifest. Now at version: " + remoteWriteOperation.getManifest().getVersion());
SignalStore.storageService().setManifest(remoteWriteOperation.getManifest());
stopwatch.split("remote-write");
needsMultiDeviceSync = true;
} else {
Log.i(TAG, "No remote writes needed. Still at version: " + remoteManifest.getVersion());
}
if (needsForcePush && SignalStore.account().isPrimaryDevice()) {
Log.w(TAG, "Scheduling a force push.");
ApplicationDependencies.getJobManager().add(new StorageForcePushJob());
}
stopwatch.stop(TAG);
return needsMultiDeviceSync;
}
use of org.thoughtcrime.securesms.database.UnknownStorageIdDatabase in project Signal-Android by WhisperSystems.
the class StorageSyncJob method buildLocalStorageRecords.
@NonNull
private static List<SignalStorageRecord> buildLocalStorageRecords(@NonNull Context context, @NonNull Recipient self, @NonNull Collection<StorageId> ids) {
if (ids.isEmpty()) {
return Collections.emptyList();
}
RecipientDatabase recipientDatabase = SignalDatabase.recipients();
UnknownStorageIdDatabase storageIdDatabase = SignalDatabase.unknownStorageIds();
List<SignalStorageRecord> records = new ArrayList<>(ids.size());
for (StorageId id : ids) {
switch(id.getType()) {
case ManifestRecord.Identifier.Type.CONTACT_VALUE:
case ManifestRecord.Identifier.Type.GROUPV1_VALUE:
case ManifestRecord.Identifier.Type.GROUPV2_VALUE:
RecipientRecord settings = recipientDatabase.getByStorageId(id.getRaw());
if (settings != null) {
if (settings.getGroupType() == RecipientDatabase.GroupType.SIGNAL_V2 && settings.getSyncExtras().getGroupMasterKey() == null) {
throw new MissingGv2MasterKeyError();
} else {
records.add(StorageSyncModels.localToRemoteRecord(settings));
}
} else {
throw new MissingRecipientModelError("Missing local recipient model! Type: " + id.getType());
}
break;
case ManifestRecord.Identifier.Type.ACCOUNT_VALUE:
if (!Arrays.equals(self.getStorageServiceId(), id.getRaw())) {
throw new AssertionError("Local storage ID doesn't match self!");
}
records.add(StorageSyncHelper.buildAccountRecord(context, self));
break;
default:
SignalStorageRecord unknown = storageIdDatabase.getById(id.getRaw());
if (unknown != null) {
records.add(unknown);
} else {
throw new MissingUnknownModelError("Missing local unknown model! Type: " + id.getType());
}
break;
}
}
return records;
}
use of org.thoughtcrime.securesms.database.UnknownStorageIdDatabase in project Signal-Android by signalapp.
the class StorageForcePushJob method onRun.
@Override
protected void onRun() throws IOException, RetryLaterException {
if (SignalStore.account().isLinkedDevice()) {
Log.i(TAG, "Only the primary device can force push");
return;
}
StorageKey storageServiceKey = SignalStore.storageService().getOrCreateStorageKey();
SignalServiceAccountManager accountManager = ApplicationDependencies.getSignalServiceAccountManager();
RecipientDatabase recipientDatabase = SignalDatabase.recipients();
UnknownStorageIdDatabase storageIdDatabase = SignalDatabase.unknownStorageIds();
long currentVersion = accountManager.getStorageManifestVersion();
Map<RecipientId, StorageId> oldContactStorageIds = recipientDatabase.getContactStorageSyncIdsMap();
long newVersion = currentVersion + 1;
Map<RecipientId, StorageId> newContactStorageIds = generateContactStorageIds(oldContactStorageIds);
List<SignalStorageRecord> inserts = Stream.of(oldContactStorageIds.keySet()).map(recipientDatabase::getRecordForSync).withoutNulls().map(s -> StorageSyncModels.localToRemoteRecord(s, Objects.requireNonNull(newContactStorageIds.get(s.getId())).getRaw())).toList();
SignalStorageRecord accountRecord = StorageSyncHelper.buildAccountRecord(context, Recipient.self().fresh());
List<StorageId> allNewStorageIds = new ArrayList<>(newContactStorageIds.values());
inserts.add(accountRecord);
allNewStorageIds.add(accountRecord.getId());
SignalStorageManifest manifest = new SignalStorageManifest(newVersion, allNewStorageIds);
StorageSyncValidations.validateForcePush(manifest, inserts, Recipient.self().fresh());
try {
if (newVersion > 1) {
Log.i(TAG, String.format(Locale.ENGLISH, "Force-pushing data. Inserting %d IDs.", inserts.size()));
if (accountManager.resetStorageRecords(storageServiceKey, manifest, inserts).isPresent()) {
Log.w(TAG, "Hit a conflict. Trying again.");
throw new RetryLaterException();
}
} else {
Log.i(TAG, String.format(Locale.ENGLISH, "First version, normal push. Inserting %d IDs.", inserts.size()));
if (accountManager.writeStorageRecords(storageServiceKey, manifest, inserts, Collections.emptyList()).isPresent()) {
Log.w(TAG, "Hit a conflict. Trying again.");
throw new RetryLaterException();
}
}
} catch (InvalidKeyException e) {
Log.w(TAG, "Hit an invalid key exception, which likely indicates a conflict.");
throw new RetryLaterException(e);
}
Log.i(TAG, "Force push succeeded. Updating local manifest version to: " + newVersion);
SignalStore.storageService().setManifest(manifest);
recipientDatabase.applyStorageIdUpdates(newContactStorageIds);
recipientDatabase.applyStorageIdUpdates(Collections.singletonMap(Recipient.self().getId(), accountRecord.getId()));
storageIdDatabase.deleteAll();
}
use of org.thoughtcrime.securesms.database.UnknownStorageIdDatabase in project Signal-Android by signalapp.
the class StorageSyncJob method performSync.
private boolean performSync() throws IOException, RetryLaterException, InvalidKeyException {
final Stopwatch stopwatch = new Stopwatch("StorageSync");
final SQLiteDatabase db = SignalDatabase.getRawDatabase();
final SignalServiceAccountManager accountManager = ApplicationDependencies.getSignalServiceAccountManager();
final UnknownStorageIdDatabase storageIdDatabase = SignalDatabase.unknownStorageIds();
final StorageKey storageServiceKey = SignalStore.storageService().getOrCreateStorageKey();
final SignalStorageManifest localManifest = SignalStore.storageService().getManifest();
final SignalStorageManifest remoteManifest = accountManager.getStorageManifestIfDifferentVersion(storageServiceKey, localManifest.getVersion()).or(localManifest);
stopwatch.split("remote-manifest");
Recipient self = freshSelf();
boolean needsMultiDeviceSync = false;
boolean needsForcePush = false;
if (self.getStorageServiceId() == null) {
Log.w(TAG, "No storageId for self. Generating.");
SignalDatabase.recipients().updateStorageId(self.getId(), StorageSyncHelper.generateKey());
self = freshSelf();
}
Log.i(TAG, "Our version: " + localManifest.getVersion() + ", their version: " + remoteManifest.getVersion());
if (remoteManifest.getVersion() > localManifest.getVersion()) {
Log.i(TAG, "[Remote Sync] Newer manifest version found!");
List<StorageId> localStorageIdsBeforeMerge = getAllLocalStorageIds(context, self);
IdDifferenceResult idDifference = StorageSyncHelper.findIdDifference(remoteManifest.getStorageIds(), localStorageIdsBeforeMerge);
if (idDifference.hasTypeMismatches() && SignalStore.account().isPrimaryDevice()) {
Log.w(TAG, "[Remote Sync] Found type mismatches in the ID sets! Scheduling a force push after this sync completes.");
needsForcePush = true;
}
Log.i(TAG, "[Remote Sync] Pre-Merge ID Difference :: " + idDifference);
stopwatch.split("remote-id-diff");
if (!idDifference.isEmpty()) {
Log.i(TAG, "[Remote Sync] Retrieving records for key difference.");
List<SignalStorageRecord> remoteOnly = accountManager.readStorageRecords(storageServiceKey, idDifference.getRemoteOnlyIds());
stopwatch.split("remote-records");
if (remoteOnly.size() != idDifference.getRemoteOnlyIds().size()) {
Log.w(TAG, "[Remote Sync] Could not find all remote-only records! Requested: " + idDifference.getRemoteOnlyIds().size() + ", Found: " + remoteOnly.size() + ". These stragglers should naturally get deleted during the sync.");
}
List<SignalContactRecord> remoteContacts = new LinkedList<>();
List<SignalGroupV1Record> remoteGv1 = new LinkedList<>();
List<SignalGroupV2Record> remoteGv2 = new LinkedList<>();
List<SignalAccountRecord> remoteAccount = new LinkedList<>();
List<SignalStorageRecord> remoteUnknown = new LinkedList<>();
for (SignalStorageRecord remote : remoteOnly) {
if (remote.getContact().isPresent()) {
remoteContacts.add(remote.getContact().get());
} else if (remote.getGroupV1().isPresent()) {
remoteGv1.add(remote.getGroupV1().get());
} else if (remote.getGroupV2().isPresent()) {
remoteGv2.add(remote.getGroupV2().get());
} else if (remote.getAccount().isPresent()) {
remoteAccount.add(remote.getAccount().get());
} else if (remote.getId().isUnknown()) {
remoteUnknown.add(remote);
} else {
Log.w(TAG, "Bad record! Type is a known value (" + remote.getId().getType() + "), but doesn't have a matching inner record. Dropping it.");
}
}
db.beginTransaction();
try {
self = freshSelf();
Log.i(TAG, "[Remote Sync] Remote-Only :: Contacts: " + remoteContacts.size() + ", GV1: " + remoteGv1.size() + ", GV2: " + remoteGv2.size() + ", Account: " + remoteAccount.size());
new ContactRecordProcessor(context, self).process(remoteContacts, StorageSyncHelper.KEY_GENERATOR);
new GroupV1RecordProcessor(context).process(remoteGv1, StorageSyncHelper.KEY_GENERATOR);
new GroupV2RecordProcessor(context).process(remoteGv2, StorageSyncHelper.KEY_GENERATOR);
self = freshSelf();
new AccountRecordProcessor(context, self).process(remoteAccount, StorageSyncHelper.KEY_GENERATOR);
List<SignalStorageRecord> unknownInserts = remoteUnknown;
List<StorageId> unknownDeletes = Stream.of(idDifference.getLocalOnlyIds()).filter(StorageId::isUnknown).toList();
Log.i(TAG, "[Remote Sync] Unknowns :: " + unknownInserts.size() + " inserts, " + unknownDeletes.size() + " deletes");
storageIdDatabase.insert(unknownInserts);
storageIdDatabase.delete(unknownDeletes);
db.setTransactionSuccessful();
} finally {
db.endTransaction();
ApplicationDependencies.getDatabaseObserver().notifyConversationListListeners();
stopwatch.split("remote-merge-transaction");
}
} else {
Log.i(TAG, "[Remote Sync] Remote version was newer, but there were no remote-only IDs.");
}
} else if (remoteManifest.getVersion() < localManifest.getVersion()) {
Log.w(TAG, "[Remote Sync] Remote version was older. User might have switched accounts.");
}
if (remoteManifest != localManifest) {
Log.i(TAG, "[Remote Sync] Saved new manifest. Now at version: " + remoteManifest.getVersion());
SignalStore.storageService().setManifest(remoteManifest);
}
Log.i(TAG, "We are up-to-date with the remote storage state.");
final WriteOperationResult remoteWriteOperation;
db.beginTransaction();
try {
self = freshSelf();
List<StorageId> localStorageIds = getAllLocalStorageIds(context, self);
IdDifferenceResult idDifference = StorageSyncHelper.findIdDifference(remoteManifest.getStorageIds(), localStorageIds);
List<SignalStorageRecord> remoteInserts = buildLocalStorageRecords(context, self, idDifference.getLocalOnlyIds());
List<byte[]> remoteDeletes = Stream.of(idDifference.getRemoteOnlyIds()).map(StorageId::getRaw).toList();
Log.i(TAG, "ID Difference :: " + idDifference);
remoteWriteOperation = new WriteOperationResult(new SignalStorageManifest(remoteManifest.getVersion() + 1, localStorageIds), remoteInserts, remoteDeletes);
db.setTransactionSuccessful();
} finally {
db.endTransaction();
stopwatch.split("local-data-transaction");
}
if (!remoteWriteOperation.isEmpty()) {
Log.i(TAG, "We have something to write remotely.");
Log.i(TAG, "WriteOperationResult :: " + remoteWriteOperation);
StorageSyncValidations.validate(remoteWriteOperation, remoteManifest, needsForcePush, self);
Optional<SignalStorageManifest> conflict = accountManager.writeStorageRecords(storageServiceKey, remoteWriteOperation.getManifest(), remoteWriteOperation.getInserts(), remoteWriteOperation.getDeletes());
if (conflict.isPresent()) {
Log.w(TAG, "Hit a conflict when trying to resolve the conflict! Retrying.");
throw new RetryLaterException();
}
Log.i(TAG, "Saved new manifest. Now at version: " + remoteWriteOperation.getManifest().getVersion());
SignalStore.storageService().setManifest(remoteWriteOperation.getManifest());
stopwatch.split("remote-write");
needsMultiDeviceSync = true;
} else {
Log.i(TAG, "No remote writes needed. Still at version: " + remoteManifest.getVersion());
}
if (needsForcePush && SignalStore.account().isPrimaryDevice()) {
Log.w(TAG, "Scheduling a force push.");
ApplicationDependencies.getJobManager().add(new StorageForcePushJob());
}
stopwatch.stop(TAG);
return needsMultiDeviceSync;
}
use of org.thoughtcrime.securesms.database.UnknownStorageIdDatabase in project Signal-Android by WhisperSystems.
the class StorageForcePushJob method onRun.
@Override
protected void onRun() throws IOException, RetryLaterException {
if (SignalStore.account().isLinkedDevice()) {
Log.i(TAG, "Only the primary device can force push");
return;
}
StorageKey storageServiceKey = SignalStore.storageService().getOrCreateStorageKey();
SignalServiceAccountManager accountManager = ApplicationDependencies.getSignalServiceAccountManager();
RecipientDatabase recipientDatabase = SignalDatabase.recipients();
UnknownStorageIdDatabase storageIdDatabase = SignalDatabase.unknownStorageIds();
long currentVersion = accountManager.getStorageManifestVersion();
Map<RecipientId, StorageId> oldContactStorageIds = recipientDatabase.getContactStorageSyncIdsMap();
long newVersion = currentVersion + 1;
Map<RecipientId, StorageId> newContactStorageIds = generateContactStorageIds(oldContactStorageIds);
List<SignalStorageRecord> inserts = Stream.of(oldContactStorageIds.keySet()).map(recipientDatabase::getRecordForSync).withoutNulls().map(s -> StorageSyncModels.localToRemoteRecord(s, Objects.requireNonNull(newContactStorageIds.get(s.getId())).getRaw())).toList();
SignalStorageRecord accountRecord = StorageSyncHelper.buildAccountRecord(context, Recipient.self().fresh());
List<StorageId> allNewStorageIds = new ArrayList<>(newContactStorageIds.values());
inserts.add(accountRecord);
allNewStorageIds.add(accountRecord.getId());
SignalStorageManifest manifest = new SignalStorageManifest(newVersion, allNewStorageIds);
StorageSyncValidations.validateForcePush(manifest, inserts, Recipient.self().fresh());
try {
if (newVersion > 1) {
Log.i(TAG, String.format(Locale.ENGLISH, "Force-pushing data. Inserting %d IDs.", inserts.size()));
if (accountManager.resetStorageRecords(storageServiceKey, manifest, inserts).isPresent()) {
Log.w(TAG, "Hit a conflict. Trying again.");
throw new RetryLaterException();
}
} else {
Log.i(TAG, String.format(Locale.ENGLISH, "First version, normal push. Inserting %d IDs.", inserts.size()));
if (accountManager.writeStorageRecords(storageServiceKey, manifest, inserts, Collections.emptyList()).isPresent()) {
Log.w(TAG, "Hit a conflict. Trying again.");
throw new RetryLaterException();
}
}
} catch (InvalidKeyException e) {
Log.w(TAG, "Hit an invalid key exception, which likely indicates a conflict.");
throw new RetryLaterException(e);
}
Log.i(TAG, "Force push succeeded. Updating local manifest version to: " + newVersion);
SignalStore.storageService().setManifest(manifest);
recipientDatabase.applyStorageIdUpdates(newContactStorageIds);
recipientDatabase.applyStorageIdUpdates(Collections.singletonMap(Recipient.self().getId(), accountRecord.getId()));
storageIdDatabase.deleteAll();
}
Aggregations