use of com.instaclustr.esop.impl.interaction.CassandraSchemaVersion in project esop by instaclustr.
the class BaseBackupOperationCoordinator method coordinate.
@Override
public void coordinate(final Operation<BackupOperationRequest> operation) {
final BackupOperationRequest request = operation.request;
logger.info(request.toString());
try {
assert cassandraJMXService != null;
assert backuperFactoryMap != null;
assert bucketServiceFactoryMap != null;
assert objectMapper != null;
if (!request.skipBucketVerification) {
try (final BucketService bucketService = bucketServiceFactoryMap.get(request.storageLocation.storageProvider).createBucketService(request)) {
bucketService.checkBucket(request.storageLocation.bucket, request.createMissingBucket);
}
}
final CassandraData cassandraData = CassandraData.parse(request.dataDirs.get(0));
cassandraData.setDatabaseEntitiesFromRequest(request.entities);
final List<String> tokens = new CassandraTokens(cassandraJMXService).act();
logger.info("Tokens {}", tokens);
if (!Snapshots.snapshotContainsTimestamp(operation.request.snapshotTag)) {
if (operation.request.schemaVersion == null) {
operation.request.schemaVersion = new CassandraSchemaVersion(cassandraJMXService).act();
}
operation.request.snapshotTag = resolveSnapshotTag(operation.request, System.currentTimeMillis());
}
logger.info("Taking snapshot with name {}", request.snapshotTag);
new TakeSnapshotOperation(cassandraJMXService, new TakeSnapshotOperationRequest(request.entities, request.snapshotTag), cassandraVersionProvider).run0();
Snapshots.hashSpec = hashSpec;
final Snapshots snapshots = Snapshots.parse(request.dataDirs, request.snapshotTag);
final Optional<Snapshot> snapshot = snapshots.get(request.snapshotTag);
if (!snapshot.isPresent()) {
throw new IllegalStateException(format("There is not any snapshot of tag %s", request.snapshotTag));
}
final Manifest manifest = Manifest.from(snapshot.get());
manifest.setSchemaVersion(request.schemaVersion);
manifest.setTokens(tokens);
// manifest
final Path localManifestPath = getLocalManifestPath(request.snapshotTag);
Manifest.write(manifest, localManifestPath, objectMapper);
manifest.setManifest(getManifestAsManifestEntry(localManifestPath));
try (final Backuper backuper = backuperFactoryMap.get(request.storageLocation.storageProvider).createBackuper(request)) {
final List<ManifestEntry> manifestEntries = manifest.getManifestEntries();
Session<UploadUnit> uploadSession = null;
try {
uploadSession = uploadTracker.submit(backuper, operation, manifestEntries, request.snapshotTag, operation.request.concurrentConnections);
uploadSession.waitUntilConsideredFinished();
uploadTracker.cancelIfNecessary(uploadSession);
final List<UploadUnit> failedUnits = uploadSession.getFailedUnits();
if (!failedUnits.isEmpty()) {
final String message = failedUnits.stream().map(unit -> unit.getManifestEntry().objectKey.toString()).collect(Collectors.joining(","));
logger.error(message);
throw new IOException(format("Unable to upload some files successfully: %s", message));
}
} finally {
uploadTracker.removeSession(uploadSession);
uploadSession = null;
}
if (operation.request.uploadClusterTopology) {
// here we will upload all topology because we do not know what restore might look like (what dc a restorer will restore against if any)
final ClusterTopology topology = new CassandraClusterTopology(cassandraJMXService, null).act();
ClusterTopology.upload(backuper, topology, objectMapper, operation.request.snapshotTag);
}
} finally {
manifest.cleanup();
}
} catch (final Exception ex) {
operation.addError(Error.from(ex));
} finally {
final ClearSnapshotOperation cso = new ClearSnapshotOperation(cassandraJMXService, new ClearSnapshotOperationRequest(request.snapshotTag));
try {
cso.run0();
} catch (final Exception ex) {
operation.addErrors(cso.errors);
}
}
}
use of com.instaclustr.esop.impl.interaction.CassandraSchemaVersion in project esop by instaclustr.
the class ManifestTest method getManifest.
private Manifest getManifest(String snapshotName) throws Exception {
waitForOperation(new TakeSnapshotOperation(jmx, new TakeSnapshotOperationRequest(DatabaseEntities.empty(), snapshotName), cassandraVersionProvider));
Snapshots snapshot = Snapshots.parse(cassandraDataDir);
Manifest manifest = new Manifest(snapshot.get(snapshotName).get());
manifest.setSchemaVersion(new CassandraSchemaVersion(jmx).act());
manifest.setTokens(new CassandraTokens(jmx).act());
Thread.sleep(5000);
return manifest;
}
use of com.instaclustr.esop.impl.interaction.CassandraSchemaVersion in project esop by instaclustr.
the class AbstractBackupTest method liveBackupWithRestoreOnDifferentSchema.
public void liveBackupWithRestoreOnDifferentSchema(final String[][] arguments, final String cassandraVersion) throws Exception {
Cassandra cassandra = getCassandra(cassandraDir, cassandraVersion);
cassandra.start();
waitForCql();
try (CqlSession session = CqlSession.builder().build()) {
createTable(session, KEYSPACE, TABLE);
createTable(session, KEYSPACE_2, TABLE_2);
// after this, table and table2 will contain 2 rows each
insert(2, session, new ArrayList<String[]>() {
{
add(new String[] { KEYSPACE, TABLE });
add(new String[] { KEYSPACE_2, TABLE_2 });
}
});
// first backup
Esop.mainWithoutExit(arguments[0]);
String firstSchemaVersion = new CassandraSchemaVersion(new CassandraJMXServiceImpl(new CassandraJMXConnectionInfo())).act();
// create third schema, by this way, Cassandra schema will change
createTable(session, KEYSPACE_3, TABLE_3);
waitUntilSchemaChanged(firstSchemaVersion);
// after this, table and table2 will contain 4 rows each
// and table3 just 2
insert(2, session, new ArrayList<String[]>() {
{
add(new String[] { KEYSPACE, TABLE });
add(new String[] { KEYSPACE_2, TABLE_2 });
add(new String[] { KEYSPACE_3, TABLE_3 });
}
});
// second backup
Esop.mainWithoutExit(arguments[1]);
// here we want to restore into first snapshot even though schema has changed, because we added the third table
// (keep in mind we have not changed schema of any table, we changed Cassandra schema as such)
// restore into the first backup
// download
Esop.mainWithoutExit(arguments[2]);
// truncate
Esop.mainWithoutExit(arguments[3]);
// after truncating, we see that we have truncated just two tables which were
// in snapshot from snapshot1, not the third one
dumpTableAndAssertRowCount(session, KEYSPACE, TABLE, 0);
dumpTableAndAssertRowCount(session, KEYSPACE_2, TABLE_2, 0);
dumpTableAndAssertRowCount(session, KEYSPACE_3, TABLE_3, 2);
// import
Esop.mainWithoutExit(arguments[4]);
// cleanup
Esop.mainWithoutExit(arguments[5]);
// verify
// here we check that table1 and table2 contains 2 rows each (as we restored it from the first snapshot) and table 3 will contain still 2
dumpTableAndAssertRowCount(session, KEYSPACE, TABLE, 2);
dumpTableAndAssertRowCount(session, KEYSPACE_2, TABLE_2, 2);
dumpTableAndAssertRowCount(session, KEYSPACE_3, TABLE_3, 2);
} finally {
cassandra.stop();
FileUtils.deleteDirectory(cassandraDir);
deleteDirectory(Paths.get(target("backup1")));
}
}
use of com.instaclustr.esop.impl.interaction.CassandraSchemaVersion in project esop by instaclustr.
the class AbstractBackupTest method liveBackupWithRestoreOnDifferentTableSchema.
public void liveBackupWithRestoreOnDifferentTableSchema(final String[][] arguments, final String cassandraVersion, final boolean tableAddition) throws Exception {
Cassandra cassandra = getCassandra(cassandraDir, cassandraVersion);
cassandra.start();
waitForCql();
try (CqlSession session = CqlSession.builder().build()) {
createTable(session, KEYSPACE, TABLE);
createTable(session, KEYSPACE_2, TABLE_2);
// after this, table and table2 will contain 2 rows each
insert(2, session, new ArrayList<String[]>() {
{
add(new String[] { KEYSPACE, TABLE });
add(new String[] { KEYSPACE_2, TABLE_2 });
}
});
// first backup
Esop.mainWithoutExit(arguments[0]);
String firstSchemaVersion = new CassandraSchemaVersion(new CassandraJMXServiceImpl(new CassandraJMXConnectionInfo())).act();
// create third schema, by this way, Cassandra schema will change
createTable(session, KEYSPACE_3, TABLE_3);
waitUntilSchemaChanged(firstSchemaVersion);
// after this, table and table2 will contain 4 rows each
// and table3 just 2
insert(2, session, new ArrayList<String[]>() {
{
add(new String[] { KEYSPACE, TABLE });
add(new String[] { KEYSPACE_2, TABLE_2 });
add(new String[] { KEYSPACE_3, TABLE_3 });
}
});
// second backup
Esop.mainWithoutExit(arguments[1]);
if (tableAddition) {
addColumnToTable(session, KEYSPACE, TABLE, "newColumn", TEXT);
} else {
removeColumnFromTable(session, KEYSPACE, TABLE, TestEntity.NAME);
}
// restore into the first snapshot where table1 was without newly added column
// we effectively restored SSTables on different schema so we expect that values in the new column will be "null"
// download
Esop.mainWithoutExit(arguments[2]);
// truncate
Esop.mainWithoutExit(arguments[3]);
// after truncating, we see that we have truncated just two tables which were
// in snapshot from snapshot1, not the third one
dumpTableAndAssertRowCount(session, KEYSPACE, TABLE, 0);
dumpTableAndAssertRowCount(session, KEYSPACE_2, TABLE_2, 0);
dumpTableAndAssertRowCount(session, KEYSPACE_3, TABLE_3, 2);
// import
Esop.mainWithoutExit(arguments[4]);
// cleanup
Esop.mainWithoutExit(arguments[5]);
// verify
// here we check that table1 and table2 contains 2 rows each (as we restored it from the first snapshot) and table 3 will contain still 2
dumpTableAndAssertRowCount(session, KEYSPACE, TABLE, 2);
dumpTableAndAssertRowCount(session, KEYSPACE_2, TABLE_2, 2);
dumpTableAndAssertRowCount(session, KEYSPACE_3, TABLE_3, 2);
} finally {
cassandra.stop();
FileUtils.deleteDirectory(cassandraDir);
deleteDirectory(Paths.get(target("backup1")));
}
}
use of com.instaclustr.esop.impl.interaction.CassandraSchemaVersion in project esop by instaclustr.
the class ManifestTest method testJsonManifest.
@Test
public void testJsonManifest() throws Exception {
try {
final List<String> tokens = new CassandraTokens(jmx).act();
DatabaseEntities databaseEntities = DatabaseEntities.empty();
// first table
createTable("ks1", "ks1t1");
disableAutocompaction("ks1");
// #1 insert and flush & take snapshot
insertDataIntoTable("ks1", "ks1t1");
flush("ks1");
insertDataIntoTable("ks1", "ks1t1");
flush("ks1");
waitForOperation(new TakeSnapshotOperation(jmx, new TakeSnapshotOperationRequest(databaseEntities, "snapshot1"), cassandraVersionProvider));
// #2 insert and flush & take snapshot
insertDataIntoTable("ks1", "ks1t1");
flush("ks1");
insertDataIntoTable("ks1", "ks1t1");
flush("ks1");
waitForOperation(new TakeSnapshotOperation(jmx, new TakeSnapshotOperationRequest(databaseEntities, "snapshot2"), cassandraVersionProvider));
// second table
createTable("ks2", "ks2t1");
disableAutocompaction("ks2");
// #1 insert and flush & take snapshot
insertDataIntoTable("ks2", "ks2t1");
flush("ks2");
insertDataIntoTable("ks2", "ks2t1");
flush("ks2");
waitForOperation(new TakeSnapshotOperation(jmx, new TakeSnapshotOperationRequest(databaseEntities, "snapshot3"), cassandraVersionProvider));
// parse
final Snapshots snapshots = Snapshots.parse(cassandraDataDir);
assertNotNull(snapshots);
assertFalse(snapshots.isEmpty());
assertEquals(3, snapshots.size());
assertTrue(snapshots.get("snapshot1").isPresent());
assertTrue(snapshots.get("snapshot2").isPresent());
assertTrue(snapshots.get("snapshot3").isPresent());
Manifest manifest = new Manifest(snapshots.get("snapshot3").get());
// manifest itself, but it wont be serialised
final Path localManifestPath = getLocalManifestPath("snapshot1");
manifest.setManifest(getManifestAsManifestEntry(localManifestPath));
// tokens
manifest.setTokens(tokens);
final String schemaVersion = new CassandraSchemaVersion(jmx).act();
manifest.setSchemaVersion(schemaVersion);
String writtenManifestAsJson = Manifest.write(manifest, objectMapper);
logger.info(writtenManifestAsJson);
assertNotNull(writtenManifestAsJson);
Snapshot snapshot3 = snapshots.get("snapshot3").get();
Optional<Keyspace> ks2 = snapshot3.getKeyspace("ks2");
assertTrue(ks2.isPresent());
assertTrue(ks2.get().containsTable("ks2t1"));
List<ManifestEntry> ks2t1 = ks2.get().getManifestEntries("ks2t1");
assertFalse(ks2t1.isEmpty());
Manifest readManifest = Manifest.read(writtenManifestAsJson, objectMapper);
assertNotNull(readManifest);
HashMultimap<String, String> ksAndTables = readManifest.getSnapshot().getKeyspacesAndTables();
// also system
assertTrue(ksAndTables.size() > 2);
assertTrue(ksAndTables.containsEntry("ks1", "ks1t1"));
assertTrue(ksAndTables.containsEntry("ks2", "ks2t1"));
HashMultimap<String, String> ksAndTablesWithoutSystem = readManifest.getSnapshot().getKeyspacesAndTables(false);
assertEquals(2, ksAndTablesWithoutSystem.size());
assertTrue(ksAndTables.containsEntry("ks1", "ks1t1"));
assertTrue(ksAndTables.containsEntry("ks2", "ks2t1"));
snapshots.clear();
assertTrue(snapshots.isEmpty());
assertEquals(0, snapshots.size());
} finally {
try {
waitForOperation(new ClearSnapshotOperation(jmx, new ClearSnapshotOperationRequest("snapshot1")));
waitForOperation(new ClearSnapshotOperation(jmx, new ClearSnapshotOperationRequest("snapshot2")));
waitForOperation(new ClearSnapshotOperation(jmx, new ClearSnapshotOperationRequest("snapshot3")));
} catch (final Exception ex) {
logger.error("Unable to clear snapshots", ex);
}
}
}
Aggregations