use of com.instaclustr.esop.impl.backup.coordination.ClearSnapshotOperation in project esop by instaclustr.
the class LocalBackupTest method testUploadTracker.
@Test
public void testUploadTracker() throws Exception {
final String snapshotName = UUID.randomUUID().toString();
final String snapshotName2 = UUID.randomUUID().toString();
List<Path> dataDirs = Arrays.asList(cassandraDir.toAbsolutePath().resolve("data").resolve("data"), cassandraDir.toAbsolutePath().resolve("data").resolve("data2"), cassandraDir.toAbsolutePath().resolve("data").resolve("data3"));
final BackupOperationRequest backupOperationRequest = getBackupOperationRequestForTracker(snapshotName, "test,test2", dataDirs);
final BackupOperationRequest backupOperationRequest2 = getBackupOperationRequestForTracker(snapshotName2, "test", dataDirs);
UploadTracker uploadTracker = null;
Cassandra cassandra = null;
try {
cassandra = getCassandra(cassandraDir, CASSANDRA_VERSION);
cassandra.start();
try (CqlSession session = CqlSession.builder().build()) {
assertEquals(populateDatabase(session).size(), NUMBER_OF_INSERTED_ROWS);
}
final AtomicBoolean wait = new AtomicBoolean(true);
final ListeningExecutorService finisher = new Executors.FixedTasksExecutorSupplier().get(10);
uploadTracker = new UploadTracker(finisher, operationsService, new HashSpec()) {
// override for testing purposes
@Override
public UploadUnit constructUnitToSubmit(final Backuper backuper, final ManifestEntry manifestEntry, final AtomicBoolean shouldCancel, final String snapshotTag, final HashSpec hashSpec) {
return new TestingUploadUnit(wait, backuper, manifestEntry, shouldCancel, snapshotTag, hashSpec);
}
};
final LocalFileBackuper backuper = new LocalFileBackuper(backupOperationRequest);
new TakeSnapshotOperation(jmxService, new TakeSnapshotOperationRequest(backupOperationRequest.entities, backupOperationRequest.snapshotTag), cassandraVersionProvider).run();
new TakeSnapshotOperation(jmxService, new TakeSnapshotOperationRequest(backupOperationRequest2.entities, backupOperationRequest2.snapshotTag), cassandraVersionProvider).run();
final Snapshots snapshots = Snapshots.parse(dataDirs);
final Optional<Snapshot> snapshot = snapshots.get(backupOperationRequest.snapshotTag);
final Optional<Snapshot> snapshot2 = snapshots.get(backupOperationRequest2.snapshotTag);
assert snapshot.isPresent();
assert snapshot2.isPresent();
Set<String> providers = Stream.of("file").collect(Collectors.toSet());
final BackupOperation backupOperation = new BackupOperation(operationCoordinator, providers, backupOperationRequest);
final BackupOperation backupOperation2 = new BackupOperation(operationCoordinator, providers, backupOperationRequest2);
final List<ManifestEntry> manifestEntries = Manifest.from(snapshot.get()).getManifestEntries();
final List<ManifestEntry> manifestEntries2 = Manifest.from(snapshot2.get()).getManifestEntries();
Session<UploadUnit> session = uploadTracker.submit(backuper, backupOperation, manifestEntries, backupOperation.request.snapshotTag, backupOperation.request.concurrentConnections);
final int submittedUnits1 = uploadTracker.submittedUnits.intValue();
Assert.assertEquals(manifestEntries.size(), submittedUnits1);
final Session<UploadUnit> session2 = uploadTracker.submit(backuper, backupOperation2, manifestEntries2, backupOperation.request.snapshotTag, backupOperation.request.concurrentConnections);
final int submittedUnits2 = uploadTracker.submittedUnits.intValue();
// even we submitted the second session, it does not change the number of units because session2
// wants to upload "test" but it is already going to be uploaded by session1
// we have effectively submitted only what should be submitted, no duplicates
// so it is as if "test" from session2 was not submitted at all
Assert.assertEquals(submittedUnits1, submittedUnits2);
Assert.assertEquals(manifestEntries.size(), uploadTracker.submittedUnits.intValue());
// however we have submitted two sessions in total
Assert.assertEquals(2, uploadTracker.submittedSessions.intValue());
// lets upload it now
wait.set(false);
session.waitUntilConsideredFinished();
session2.waitUntilConsideredFinished();
Assert.assertTrue(session.isConsideredFinished());
Assert.assertTrue(session.isSuccessful());
Assert.assertTrue(session.getFailedUnits().isEmpty());
Assert.assertEquals(uploadTracker.submittedUnits.intValue(), session.getUnits().size());
Assert.assertTrue(session2.isConsideredFinished());
Assert.assertTrue(session2.isSuccessful());
Assert.assertTrue(session2.getFailedUnits().isEmpty());
Assert.assertTrue(submittedUnits2 > session2.getUnits().size());
for (final UploadUnit uploadUnit : session2.getUnits()) {
Assert.assertTrue(session.getUnits().contains(uploadUnit));
}
Assert.assertTrue(uploadTracker.getUnits().isEmpty());
uploadTracker.removeSession(session);
uploadTracker.removeSession(session2);
Assert.assertTrue(session.getUnits().isEmpty());
Assert.assertTrue(session2.getUnits().isEmpty());
} catch (final Exception ex) {
ex.printStackTrace();
throw ex;
} finally {
new ClearSnapshotOperation(jmxService, new ClearSnapshotOperationRequest(backupOperationRequest.snapshotTag)).run();
if (cassandra != null) {
cassandra.stop();
}
uploadTracker.stopAsync();
uploadTracker.awaitTerminated(1, MINUTES);
uploadTracker.stopAsync();
uploadTracker.awaitTerminated(1, MINUTES);
FileUtils.deleteDirectory(Paths.get(target(backupOperationRequest.storageLocation.bucket)));
}
}
use of com.instaclustr.esop.impl.backup.coordination.ClearSnapshotOperation in project esop by instaclustr.
the class ManifestTest method testManifestMethods.
@Test
public void testManifestMethods() throws Exception {
Manifest[] manifests = prepareDatabase();
try {
// schemas across snapshots are not same
assertNotEquals(manifests[0].getSchemaVersion(), manifests[1].getSchemaVersion());
assertNotEquals(manifests[1].getSchemaVersion(), manifests[2].getSchemaVersion());
assertNotEquals(manifests[0].getSchemaVersion(), manifests[2].getSchemaVersion());
// however, this is true, because tables which are present in both are same
assertTrue(manifests[0].getSnapshot().hasSameSchemas(manifests[1].getSnapshot()));
assertTrue(manifests[0].getSnapshot().hasSameSchemas(manifests[2].getSnapshot()));
assertTrue(manifests[1].getSnapshot().hasSameSchemas(manifests[2].getSnapshot()));
// second and third snapshot are "same" on schemas even third snapshot has one table more
// but it is important that same tables are same
assertTrue(manifests[0].getSnapshot().getKeyspace("ks1").get().hasSameSchema(manifests[1].getSnapshot().getKeyspace("ks1").get()));
assertTrue(manifests[1].getSnapshot().getKeyspace("ks2").get().hasSameSchema(manifests[2].getSnapshot().getKeyspace("ks2").get()));
// however, we have altered a schema of k2t2
assertFalse(manifests[2].getSnapshot().hasSameSchemas(manifests[3].getSnapshot()));
assertFalse(manifests[2].getSnapshot().getKeyspace("ks2").get().hasSameSchema(manifests[3].getSnapshot().getKeyspace("ks2").get()));
Keyspace ks2BeforeAlter = manifests[2].getSnapshot().getKeyspace("ks2").get();
Keyspace ks2AfterAlter = manifests[3].getSnapshot().getKeyspace("ks2").get();
// TODO - get entities with different schemas
List<String> tablesWithDifferentSchemas = ks2BeforeAlter.getTablesWithDifferentSchemas(ks2AfterAlter);
assertFalse(tablesWithDifferentSchemas.isEmpty());
assertEquals(1, tablesWithDifferentSchemas.size());
assertEquals("ks2t2", tablesWithDifferentSchemas.get(0));
} finally {
try {
for (Manifest manifest : manifests) {
waitForOperation(new ClearSnapshotOperation(jmx, new ClearSnapshotOperationRequest(manifest.getSnapshot().getName())));
}
} catch (final Exception ex) {
logger.error("Unable to clear snapshots", ex);
}
}
}
use of com.instaclustr.esop.impl.backup.coordination.ClearSnapshotOperation in project esop by instaclustr.
the class ManifestTest method testJsonManifest.
@Test
public void testJsonManifest() throws Exception {
try {
final List<String> tokens = new CassandraTokens(jmx).act();
DatabaseEntities databaseEntities = DatabaseEntities.empty();
// first table
createTable("ks1", "ks1t1");
disableAutocompaction("ks1");
// #1 insert and flush & take snapshot
insertDataIntoTable("ks1", "ks1t1");
flush("ks1");
insertDataIntoTable("ks1", "ks1t1");
flush("ks1");
waitForOperation(new TakeSnapshotOperation(jmx, new TakeSnapshotOperationRequest(databaseEntities, "snapshot1"), cassandraVersionProvider));
// #2 insert and flush & take snapshot
insertDataIntoTable("ks1", "ks1t1");
flush("ks1");
insertDataIntoTable("ks1", "ks1t1");
flush("ks1");
waitForOperation(new TakeSnapshotOperation(jmx, new TakeSnapshotOperationRequest(databaseEntities, "snapshot2"), cassandraVersionProvider));
// second table
createTable("ks2", "ks2t1");
disableAutocompaction("ks2");
// #1 insert and flush & take snapshot
insertDataIntoTable("ks2", "ks2t1");
flush("ks2");
insertDataIntoTable("ks2", "ks2t1");
flush("ks2");
waitForOperation(new TakeSnapshotOperation(jmx, new TakeSnapshotOperationRequest(databaseEntities, "snapshot3"), cassandraVersionProvider));
// parse
final Snapshots snapshots = Snapshots.parse(cassandraDataDir);
assertNotNull(snapshots);
assertFalse(snapshots.isEmpty());
assertEquals(3, snapshots.size());
assertTrue(snapshots.get("snapshot1").isPresent());
assertTrue(snapshots.get("snapshot2").isPresent());
assertTrue(snapshots.get("snapshot3").isPresent());
Manifest manifest = new Manifest(snapshots.get("snapshot3").get());
// manifest itself, but it wont be serialised
final Path localManifestPath = getLocalManifestPath("snapshot1");
manifest.setManifest(getManifestAsManifestEntry(localManifestPath));
// tokens
manifest.setTokens(tokens);
final String schemaVersion = new CassandraSchemaVersion(jmx).act();
manifest.setSchemaVersion(schemaVersion);
String writtenManifestAsJson = Manifest.write(manifest, objectMapper);
logger.info(writtenManifestAsJson);
assertNotNull(writtenManifestAsJson);
Snapshot snapshot3 = snapshots.get("snapshot3").get();
Optional<Keyspace> ks2 = snapshot3.getKeyspace("ks2");
assertTrue(ks2.isPresent());
assertTrue(ks2.get().containsTable("ks2t1"));
List<ManifestEntry> ks2t1 = ks2.get().getManifestEntries("ks2t1");
assertFalse(ks2t1.isEmpty());
Manifest readManifest = Manifest.read(writtenManifestAsJson, objectMapper);
assertNotNull(readManifest);
HashMultimap<String, String> ksAndTables = readManifest.getSnapshot().getKeyspacesAndTables();
// also system
assertTrue(ksAndTables.size() > 2);
assertTrue(ksAndTables.containsEntry("ks1", "ks1t1"));
assertTrue(ksAndTables.containsEntry("ks2", "ks2t1"));
HashMultimap<String, String> ksAndTablesWithoutSystem = readManifest.getSnapshot().getKeyspacesAndTables(false);
assertEquals(2, ksAndTablesWithoutSystem.size());
assertTrue(ksAndTables.containsEntry("ks1", "ks1t1"));
assertTrue(ksAndTables.containsEntry("ks2", "ks2t1"));
snapshots.clear();
assertTrue(snapshots.isEmpty());
assertEquals(0, snapshots.size());
} finally {
try {
waitForOperation(new ClearSnapshotOperation(jmx, new ClearSnapshotOperationRequest("snapshot1")));
waitForOperation(new ClearSnapshotOperation(jmx, new ClearSnapshotOperationRequest("snapshot2")));
waitForOperation(new ClearSnapshotOperation(jmx, new ClearSnapshotOperationRequest("snapshot3")));
} catch (final Exception ex) {
logger.error("Unable to clear snapshots", ex);
}
}
}
Aggregations