use of com.instaclustr.esop.impl.CassandraData in project esop by instaclustr.
the class BaseBackupOperationCoordinator method coordinate.
@Override
public void coordinate(final Operation<BackupOperationRequest> operation) {
final BackupOperationRequest request = operation.request;
logger.info(request.toString());
try {
assert cassandraJMXService != null;
assert backuperFactoryMap != null;
assert bucketServiceFactoryMap != null;
assert objectMapper != null;
if (!request.skipBucketVerification) {
try (final BucketService bucketService = bucketServiceFactoryMap.get(request.storageLocation.storageProvider).createBucketService(request)) {
bucketService.checkBucket(request.storageLocation.bucket, request.createMissingBucket);
}
}
final CassandraData cassandraData = CassandraData.parse(request.dataDirs.get(0));
cassandraData.setDatabaseEntitiesFromRequest(request.entities);
final List<String> tokens = new CassandraTokens(cassandraJMXService).act();
logger.info("Tokens {}", tokens);
if (!Snapshots.snapshotContainsTimestamp(operation.request.snapshotTag)) {
if (operation.request.schemaVersion == null) {
operation.request.schemaVersion = new CassandraSchemaVersion(cassandraJMXService).act();
}
operation.request.snapshotTag = resolveSnapshotTag(operation.request, System.currentTimeMillis());
}
logger.info("Taking snapshot with name {}", request.snapshotTag);
new TakeSnapshotOperation(cassandraJMXService, new TakeSnapshotOperationRequest(request.entities, request.snapshotTag), cassandraVersionProvider).run0();
Snapshots.hashSpec = hashSpec;
final Snapshots snapshots = Snapshots.parse(request.dataDirs, request.snapshotTag);
final Optional<Snapshot> snapshot = snapshots.get(request.snapshotTag);
if (!snapshot.isPresent()) {
throw new IllegalStateException(format("There is not any snapshot of tag %s", request.snapshotTag));
}
final Manifest manifest = Manifest.from(snapshot.get());
manifest.setSchemaVersion(request.schemaVersion);
manifest.setTokens(tokens);
// manifest
final Path localManifestPath = getLocalManifestPath(request.snapshotTag);
Manifest.write(manifest, localManifestPath, objectMapper);
manifest.setManifest(getManifestAsManifestEntry(localManifestPath));
try (final Backuper backuper = backuperFactoryMap.get(request.storageLocation.storageProvider).createBackuper(request)) {
final List<ManifestEntry> manifestEntries = manifest.getManifestEntries();
Session<UploadUnit> uploadSession = null;
try {
uploadSession = uploadTracker.submit(backuper, operation, manifestEntries, request.snapshotTag, operation.request.concurrentConnections);
uploadSession.waitUntilConsideredFinished();
uploadTracker.cancelIfNecessary(uploadSession);
final List<UploadUnit> failedUnits = uploadSession.getFailedUnits();
if (!failedUnits.isEmpty()) {
final String message = failedUnits.stream().map(unit -> unit.getManifestEntry().objectKey.toString()).collect(Collectors.joining(","));
logger.error(message);
throw new IOException(format("Unable to upload some files successfully: %s", message));
}
} finally {
uploadTracker.removeSession(uploadSession);
uploadSession = null;
}
if (operation.request.uploadClusterTopology) {
// here we will upload all topology because we do not know what restore might look like (what dc a restorer will restore against if any)
final ClusterTopology topology = new CassandraClusterTopology(cassandraJMXService, null).act();
ClusterTopology.upload(backuper, topology, objectMapper, operation.request.snapshotTag);
}
} finally {
manifest.cleanup();
}
} catch (final Exception ex) {
operation.addError(Error.from(ex));
} finally {
final ClearSnapshotOperation cso = new ClearSnapshotOperation(cassandraJMXService, new ClearSnapshotOperationRequest(request.snapshotTag));
try {
cso.run0();
} catch (final Exception ex) {
operation.addErrors(cso.errors);
}
}
}
use of com.instaclustr.esop.impl.CassandraData in project esop by instaclustr.
the class CassandraDataTest method testCassandraDataMissingFromInRenamingInEntitiesShouldFail.
@Test
public // --entities=ks1.tb1 --rename=ks1.tb2=ks1.tb2 -> invalid as "from" is not in entities
void testCassandraDataMissingFromInRenamingInEntitiesShouldFail() {
CassandraData parsed = new CassandraData(tableIdsMap, paths);
parsed.setDatabaseEntitiesFromRequest(DatabaseEntities.parse("ks1 . tb1"));
parsed.setRenamedEntitiesFromRequest(RenamedEntities.parse(new HashMap<String, String>() {
{
put("ks1.tb2", "ks1.tb3");
}
}));
try {
parsed.validate();
fail("setting rename entities where 'from' is missing in entities should fail");
} catch (final Exception ex) {
// ok
}
}
use of com.instaclustr.esop.impl.CassandraData in project esop by instaclustr.
the class CassandraDataTest method testCassandraData.
@Test
public void testCassandraData() {
CassandraData parsed = new CassandraData(tableIdsMap, paths);
assertTrue(parsed.getTableId("system_schema", "tables").isPresent());
assertFalse(parsed.getTableId("system_schema", "asdadsad").isPresent());
assertFalse(parsed.getTableId("sdadasd", "types").isPresent());
Optional<List<String>> systemSchemaTablesNames = parsed.getTablesNames("system_schema");
assertTrue(systemSchemaTablesNames.isPresent());
Optional<List<Path>> systemSchemaTablesPaths = parsed.getTablesPaths("system_schema");
assertTrue(systemSchemaTablesPaths.isPresent());
assertEquals(2, systemSchemaTablesPaths.get().size());
assertTrue(parsed.containsKeyspace("ks1"));
assertFalse(parsed.containsKeyspace("asdsad"));
assertTrue(parsed.containsTable("ks1", "tb1"));
assertTrue(parsed.containsTable("ks1", "tb2"));
assertFalse(parsed.containsTable("ks1", "sdada"));
assertEquals("11111111111111111111111111111111", parsed.getTableId("ks1", "tb1").get());
assertEquals("88888888888888888888888888888888", parsed.getTableId("ks2", "tb4").get());
assertFalse(parsed.getTableId("ks2", "sdadsad").isPresent());
assertTrue(parsed.getTablesNames("ks1").get().containsAll(Arrays.asList("tb1", "tb2", "tb3", "tb4")));
assertTrue(parsed.getTablesNames("ks2").get().containsAll(Arrays.asList("tb1", "tb2", "tb3", "tb4")));
assertTrue(parsed.getTablesPaths("ks1").get().containsAll(Arrays.asList(Paths.get("/var/lib/cassandra/data/data/ks1/tb1-11111111111111111111111111111111"), Paths.get("/var/lib/cassandra/data/data/ks1/tb2-22222222222222222222222222222222"), Paths.get("/var/lib/cassandra/data/data/ks1/tb3-33333333333333333333333333333333"), Paths.get("/var/lib/cassandra/data/data/ks1/tb4-44444444444444444444444444444444"))));
assertTrue(parsed.getTablePath("ks1", "tb3").get().equals(Paths.get("/var/lib/cassandra/data/data/ks1/tb3-33333333333333333333333333333333")));
assertFalse(parsed.getTablePath("ks1", "sdad").isPresent());
assertFalse(parsed.getTablePath("sdad", "sdad").isPresent());
assertEquals(Paths.get("/var/lib/cassandra/data/data/ks1"), parsed.getKeyspacePath("ks1").get());
assertEquals(3, parsed.getKeyspacePaths().size());
// database entities
DatabaseEntities databaseEntities = parsed.toDatabaseEntities(false);
assertFalse(databaseEntities.areEmpty());
assertTrue(databaseEntities.getKeyspaces().containsAll(Arrays.asList("ks1", "ks2")));
databaseEntities.getKeyspacesAndTables().entries().forEach(entry -> {
assertTrue(parsed.getTableId(entry.getKey(), entry.getValue()).isPresent());
});
assertFalse(databaseEntities.contains("system_schema"));
assertFalse(databaseEntities.contains("asdasda", "asdad"));
}
use of com.instaclustr.esop.impl.CassandraData in project esop by instaclustr.
the class CassandraDataTest method testDatabaseEntities.
@Test
public void testDatabaseEntities() throws Exception {
CassandraData parsed = new CassandraData(tableIdsMap, paths);
DatabaseEntities entities = DatabaseEntities.parse("ks1 .tb1 , kb3. tb3");
try {
parsed.setDatabaseEntitiesFromRequest(entities);
} catch (Exception ex) {
assertEquals(ex.getMessage(), "Tables [kb3.tb3] to process are not present in Cassandra.");
}
parsed.setDatabaseEntitiesFromRequest(DatabaseEntities.parse("ks1 .tb1"));
}
use of com.instaclustr.esop.impl.CassandraData in project esop by instaclustr.
the class CassandraDataTest method testCassandraDataForRenamingEntities.
@Test
public // --entities=ks1.tb1,ks1.tb2 --rename=ks1.tb2=ks1.tb1 -> invalid as "to" is in entities
void testCassandraDataForRenamingEntities() {
CassandraData parsed = new CassandraData(tableIdsMap, paths);
parsed.setDatabaseEntitiesFromRequest(DatabaseEntities.parse("ks1.tb1, ks1.tb3, ks2.tb3, ks2.tb4"));
parsed.setRenamedEntitiesFromRequest(RenamedEntities.parse(new HashMap<String, String>() {
{
put("ks1.tb1", "ks1.tb2");
put("ks2.tb3", "ks2.tb2");
}
}));
DatabaseEntities databaseEntitiesToProcessForRestore = parsed.getDatabaseEntitiesToProcessForRestore();
assertTrue(databaseEntitiesToProcessForRestore.contains("ks1", "tb2"));
assertTrue(databaseEntitiesToProcessForRestore.contains("ks2", "tb2"));
assertTrue(databaseEntitiesToProcessForRestore.contains("ks1", "tb3"));
assertTrue(databaseEntitiesToProcessForRestore.contains("ks2", "tb4"));
assertFalse(databaseEntitiesToProcessForRestore.contains("ks1", "tb1"));
assertFalse(databaseEntitiesToProcessForRestore.contains("ks2", "tb3"));
assertEquals(2, databaseEntitiesToProcessForRestore.getKeyspaces().size());
assertEquals(4, databaseEntitiesToProcessForRestore.getKeyspacesAndTables().size());
}
Aggregations