use of com.instaclustr.esop.impl.hash.HashSpec in project esop by instaclustr.
the class BaseBackupOperationCoordinator method coordinate.
@Override
public void coordinate(final Operation<BackupOperationRequest> operation) {
final BackupOperationRequest request = operation.request;
logger.info(request.toString());
try {
assert cassandraJMXService != null;
assert backuperFactoryMap != null;
assert bucketServiceFactoryMap != null;
assert objectMapper != null;
if (!request.skipBucketVerification) {
try (final BucketService bucketService = bucketServiceFactoryMap.get(request.storageLocation.storageProvider).createBucketService(request)) {
bucketService.checkBucket(request.storageLocation.bucket, request.createMissingBucket);
}
}
final CassandraData cassandraData = CassandraData.parse(request.dataDirs.get(0));
cassandraData.setDatabaseEntitiesFromRequest(request.entities);
final List<String> tokens = new CassandraTokens(cassandraJMXService).act();
logger.info("Tokens {}", tokens);
if (!Snapshots.snapshotContainsTimestamp(operation.request.snapshotTag)) {
if (operation.request.schemaVersion == null) {
operation.request.schemaVersion = new CassandraSchemaVersion(cassandraJMXService).act();
}
operation.request.snapshotTag = resolveSnapshotTag(operation.request, System.currentTimeMillis());
}
logger.info("Taking snapshot with name {}", request.snapshotTag);
new TakeSnapshotOperation(cassandraJMXService, new TakeSnapshotOperationRequest(request.entities, request.snapshotTag), cassandraVersionProvider).run0();
Snapshots.hashSpec = hashSpec;
final Snapshots snapshots = Snapshots.parse(request.dataDirs, request.snapshotTag);
final Optional<Snapshot> snapshot = snapshots.get(request.snapshotTag);
if (!snapshot.isPresent()) {
throw new IllegalStateException(format("There is not any snapshot of tag %s", request.snapshotTag));
}
final Manifest manifest = Manifest.from(snapshot.get());
manifest.setSchemaVersion(request.schemaVersion);
manifest.setTokens(tokens);
// manifest
final Path localManifestPath = getLocalManifestPath(request.snapshotTag);
Manifest.write(manifest, localManifestPath, objectMapper);
manifest.setManifest(getManifestAsManifestEntry(localManifestPath));
try (final Backuper backuper = backuperFactoryMap.get(request.storageLocation.storageProvider).createBackuper(request)) {
final List<ManifestEntry> manifestEntries = manifest.getManifestEntries();
Session<UploadUnit> uploadSession = null;
try {
uploadSession = uploadTracker.submit(backuper, operation, manifestEntries, request.snapshotTag, operation.request.concurrentConnections);
uploadSession.waitUntilConsideredFinished();
uploadTracker.cancelIfNecessary(uploadSession);
final List<UploadUnit> failedUnits = uploadSession.getFailedUnits();
if (!failedUnits.isEmpty()) {
final String message = failedUnits.stream().map(unit -> unit.getManifestEntry().objectKey.toString()).collect(Collectors.joining(","));
logger.error(message);
throw new IOException(format("Unable to upload some files successfully: %s", message));
}
} finally {
uploadTracker.removeSession(uploadSession);
uploadSession = null;
}
if (operation.request.uploadClusterTopology) {
// here we will upload all topology because we do not know what restore might look like (what dc a restorer will restore against if any)
final ClusterTopology topology = new CassandraClusterTopology(cassandraJMXService, null).act();
ClusterTopology.upload(backuper, topology, objectMapper, operation.request.snapshotTag);
}
} finally {
manifest.cleanup();
}
} catch (final Exception ex) {
operation.addError(Error.from(ex));
} finally {
final ClearSnapshotOperation cso = new ClearSnapshotOperation(cassandraJMXService, new ClearSnapshotOperationRequest(request.snapshotTag));
try {
cso.run0();
} catch (final Exception ex) {
operation.addErrors(cso.errors);
}
}
}
use of com.instaclustr.esop.impl.hash.HashSpec in project esop by instaclustr.
the class AbstractTracker method submit.
public synchronized Session<UNIT> submit(final INTERACTOR interactor, final Operation<? extends REQUEST> operation, final Collection<ManifestEntry> entries, final String snapshotTag, final int concurrentConnections) {
final Session<UNIT> currentSession = constructSession();
currentSession.setSnapshotTag(snapshotTag);
currentSession.setId(operation.id);
if (entries.isEmpty()) {
logger.info("0 files to process.");
return currentSession;
}
// we have executor service per request in order to specify maximal
// concurrent uploads, if we had one global executor, we could not "cap it".
final ListeningExecutorService executorService = new FixedTasksExecutorSupplier().get(concurrentConnections);
final Map<ListenableFuture<Void>, Unit> futures = new HashMap<>();
for (final ManifestEntry entry : entries) {
UNIT alreadySubmitted = null;
final Iterator<UNIT> it = Collections.unmodifiableList(new ArrayList<>(units)).iterator();
while (it.hasNext()) {
UNIT unit = it.next();
if (unit.getManifestEntry().objectKey.equals(entry.objectKey)) {
alreadySubmitted = unit;
break;
}
}
if (alreadySubmitted == null) {
final UNIT unit = constructUnitToSubmit(interactor, entry, operation.getShouldCancel(), snapshotTag, hashSpec);
units.add(unit);
futures.put(executorService.submit(unit), unit);
submittedUnits.incrementAndGet();
currentSession.addUnit(unit);
} else {
logger.info(String.format("Session %s skips as already submitted: %s", currentSession.getId(), alreadySubmitted.getManifestEntry().objectKey));
currentSession.addUnit(alreadySubmitted);
}
}
sessions.add(currentSession);
submittedSessions.incrementAndGet();
futures.forEach((key, value) -> key.addListener(() -> {
synchronized (sessions) {
// increment finished units across all sessions
sessions.stream().filter(s -> s.getUnits().contains(value)).forEach(s -> {
operationsService.operation(s.getId()).ifPresent(op -> {
s.finishedUnits.incrementAndGet();
logger.debug(String.format("Progress of operation %s: %s", op.id, s.getProgress()));
op.progress = s.getProgress();
});
});
units.remove(value);
}
}, finisherExecutorService));
currentSession.setExecutorService(executorService);
return currentSession;
}
use of com.instaclustr.esop.impl.hash.HashSpec in project esop by instaclustr.
the class ListApplication method run.
@Override
public void run() {
Esop.logCommandVersionInformation(spec);
final List<Module> modules = Collections.singletonList(new ListModule());
Esop.init(this, jmxSpec, new HashSpec(), request, logger, modules);
final Operation<?> operation = operationsService.submitOperationRequest(request);
await().forever().until(() -> operation.state.isTerminalState());
if (operation.state == FAILED) {
throw new IllegalStateException(format("List operation %s was not successful.", operation.id));
}
}
use of com.instaclustr.esop.impl.hash.HashSpec in project esop by instaclustr.
the class LocalBackupTest method testUploadTracker.
@Test
public void testUploadTracker() throws Exception {
final String snapshotName = UUID.randomUUID().toString();
final String snapshotName2 = UUID.randomUUID().toString();
List<Path> dataDirs = Arrays.asList(cassandraDir.toAbsolutePath().resolve("data").resolve("data"), cassandraDir.toAbsolutePath().resolve("data").resolve("data2"), cassandraDir.toAbsolutePath().resolve("data").resolve("data3"));
final BackupOperationRequest backupOperationRequest = getBackupOperationRequestForTracker(snapshotName, "test,test2", dataDirs);
final BackupOperationRequest backupOperationRequest2 = getBackupOperationRequestForTracker(snapshotName2, "test", dataDirs);
UploadTracker uploadTracker = null;
Cassandra cassandra = null;
try {
cassandra = getCassandra(cassandraDir, CASSANDRA_VERSION);
cassandra.start();
try (CqlSession session = CqlSession.builder().build()) {
assertEquals(populateDatabase(session).size(), NUMBER_OF_INSERTED_ROWS);
}
final AtomicBoolean wait = new AtomicBoolean(true);
final ListeningExecutorService finisher = new Executors.FixedTasksExecutorSupplier().get(10);
uploadTracker = new UploadTracker(finisher, operationsService, new HashSpec()) {
// override for testing purposes
@Override
public UploadUnit constructUnitToSubmit(final Backuper backuper, final ManifestEntry manifestEntry, final AtomicBoolean shouldCancel, final String snapshotTag, final HashSpec hashSpec) {
return new TestingUploadUnit(wait, backuper, manifestEntry, shouldCancel, snapshotTag, hashSpec);
}
};
final LocalFileBackuper backuper = new LocalFileBackuper(backupOperationRequest);
new TakeSnapshotOperation(jmxService, new TakeSnapshotOperationRequest(backupOperationRequest.entities, backupOperationRequest.snapshotTag), cassandraVersionProvider).run();
new TakeSnapshotOperation(jmxService, new TakeSnapshotOperationRequest(backupOperationRequest2.entities, backupOperationRequest2.snapshotTag), cassandraVersionProvider).run();
final Snapshots snapshots = Snapshots.parse(dataDirs);
final Optional<Snapshot> snapshot = snapshots.get(backupOperationRequest.snapshotTag);
final Optional<Snapshot> snapshot2 = snapshots.get(backupOperationRequest2.snapshotTag);
assert snapshot.isPresent();
assert snapshot2.isPresent();
Set<String> providers = Stream.of("file").collect(Collectors.toSet());
final BackupOperation backupOperation = new BackupOperation(operationCoordinator, providers, backupOperationRequest);
final BackupOperation backupOperation2 = new BackupOperation(operationCoordinator, providers, backupOperationRequest2);
final List<ManifestEntry> manifestEntries = Manifest.from(snapshot.get()).getManifestEntries();
final List<ManifestEntry> manifestEntries2 = Manifest.from(snapshot2.get()).getManifestEntries();
Session<UploadUnit> session = uploadTracker.submit(backuper, backupOperation, manifestEntries, backupOperation.request.snapshotTag, backupOperation.request.concurrentConnections);
final int submittedUnits1 = uploadTracker.submittedUnits.intValue();
Assert.assertEquals(manifestEntries.size(), submittedUnits1);
final Session<UploadUnit> session2 = uploadTracker.submit(backuper, backupOperation2, manifestEntries2, backupOperation.request.snapshotTag, backupOperation.request.concurrentConnections);
final int submittedUnits2 = uploadTracker.submittedUnits.intValue();
// even we submitted the second session, it does not change the number of units because session2
// wants to upload "test" but it is already going to be uploaded by session1
// we have effectively submitted only what should be submitted, no duplicates
// so it is as if "test" from session2 was not submitted at all
Assert.assertEquals(submittedUnits1, submittedUnits2);
Assert.assertEquals(manifestEntries.size(), uploadTracker.submittedUnits.intValue());
// however we have submitted two sessions in total
Assert.assertEquals(2, uploadTracker.submittedSessions.intValue());
// lets upload it now
wait.set(false);
session.waitUntilConsideredFinished();
session2.waitUntilConsideredFinished();
Assert.assertTrue(session.isConsideredFinished());
Assert.assertTrue(session.isSuccessful());
Assert.assertTrue(session.getFailedUnits().isEmpty());
Assert.assertEquals(uploadTracker.submittedUnits.intValue(), session.getUnits().size());
Assert.assertTrue(session2.isConsideredFinished());
Assert.assertTrue(session2.isSuccessful());
Assert.assertTrue(session2.getFailedUnits().isEmpty());
Assert.assertTrue(submittedUnits2 > session2.getUnits().size());
for (final UploadUnit uploadUnit : session2.getUnits()) {
Assert.assertTrue(session.getUnits().contains(uploadUnit));
}
Assert.assertTrue(uploadTracker.getUnits().isEmpty());
uploadTracker.removeSession(session);
uploadTracker.removeSession(session2);
Assert.assertTrue(session.getUnits().isEmpty());
Assert.assertTrue(session2.getUnits().isEmpty());
} catch (final Exception ex) {
ex.printStackTrace();
throw ex;
} finally {
new ClearSnapshotOperation(jmxService, new ClearSnapshotOperationRequest(backupOperationRequest.snapshotTag)).run();
if (cassandra != null) {
cassandra.stop();
}
uploadTracker.stopAsync();
uploadTracker.awaitTerminated(1, MINUTES);
uploadTracker.stopAsync();
uploadTracker.awaitTerminated(1, MINUTES);
FileUtils.deleteDirectory(Paths.get(target(backupOperationRequest.storageLocation.bucket)));
}
}
use of com.instaclustr.esop.impl.hash.HashSpec in project esop by instaclustr.
the class BackupRestoreTest method testSSTableLister.
@Test(description = "Test that the manifest is correctly constructed, includes expected files and generates checksum if necessary")
@Ignore
public void testSSTableLister() throws Exception {
// TODO not sure why this doesn't recreate things fully given its called before each test
hardResetTestDirs();
for (TestFileConfig testFileConfig : versionsToTest) {
Path backupRoot = Paths.get("/backupRoot/keyspace1");
final String keyspace = "keyspace1";
final String table1 = "table1";
final Path table1Path = tempDirs.get(testFileConfig.cassandraVersion.toString()).resolve("data/" + keyspace + "/" + table1);
Map<String, List<ManifestEntry>> sstables = SSTableUtils.getSSTables(keyspace, table1, table1Path, backupRoot.resolve(table1Path.getFileName()), new HashSpec());
final String table2 = "table2";
final Path table2Path = tempDirs.get(testFileConfig.cassandraVersion.toString()).resolve("data/" + keyspace + "/" + table2);
sstables.putAll(SSTableUtils.getSSTables(keyspace, table2, table2Path, backupRoot.resolve(table2Path.getFileName()), new HashSpec()));
Map<Path, Path> manifestMap = new HashMap<>();
for (ManifestEntry e : sstables.values().stream().flatMap(Collection::stream).collect(Collectors.toList())) {
manifestMap.put(e.localFile, e.objectKey);
}
if (CassandraVersion.isTwoZero(testFileConfig.cassandraVersion)) {
// table1 is un-compressed so should have written out a sha1 digest
final Path localPath1 = table1Path.resolve(String.format("%s-1-big-Data.db", testFileConfig.getSstablePrefix(keyspace, table1)));
assertEquals(manifestMap.get(localPath1), backupRoot.resolve(String.format("%s/1-%s/%s-1-big-Data.db", table1, sha1Hash, testFileConfig.getSstablePrefix(keyspace, table1))));
final Path localPath2 = table1Path.resolve(String.format("%s-3-big-Index.db", testFileConfig.getSstablePrefix(keyspace, table1)));
final String checksum2 = SSTableUtils.calculateChecksum(localPath2);
assertEquals(manifestMap.get(localPath2), backupRoot.resolve(String.format("%s/3-%s/%s-3-big-Index.db", table1, checksum2, testFileConfig.getSstablePrefix(keyspace, table1))));
final Path localPath3 = table2Path.resolve(String.format("%s-1-big-Data.db", testFileConfig.getSstablePrefix(keyspace, table2)));
final String checksum3 = SSTableUtils.calculateChecksum(localPath3);
assertEquals(manifestMap.get(localPath3), backupRoot.resolve(String.format("%s/1-%s/%s-1-big-Data.db", table2, checksum3, testFileConfig.getSstablePrefix(keyspace, table2))));
assertNull(manifestMap.get(table2Path.resolve(String.format("%s-3-big-Index.db", testFileConfig.getSstablePrefix(keyspace, table2)))));
} else {
Path resolve = table1Path.resolve(String.format("%s-1-big-Data.db", testFileConfig.getSstablePrefix(keyspace, table1)));
assertEquals(manifestMap.get(resolve), backupRoot.resolve(String.format("%s/1-1000000000/%s-1-big-Data.db", table1, testFileConfig.getSstablePrefix(keyspace, table1))));
// Cassandra doesn't create CRC32 file for 2.0.x
assertEquals(manifestMap.get(table1Path.resolve(String.format("%s-2-big-Digest.crc32", testFileConfig.getSstablePrefix(keyspace, table1)))), backupRoot.resolve(String.format("%s/2-1000000000/%s-2-big-Digest.crc32", table1, testFileConfig.getSstablePrefix(keyspace, table1))));
assertEquals(manifestMap.get(table1Path.resolve(String.format("%s-3-big-Index.db", testFileConfig.getSstablePrefix(keyspace, table1)))), backupRoot.resolve(String.format("%s/3-1000000000/%s-3-big-Index.db", table1, testFileConfig.getSstablePrefix(keyspace, table1))));
assertEquals(manifestMap.get(table2Path.resolve(String.format("%s-1-big-Data.db", testFileConfig.getSstablePrefix(keyspace, table2)))), backupRoot.resolve(String.format("%s/1-1000000000/%s-1-big-Data.db", table2, testFileConfig.getSstablePrefix(keyspace, table2))));
assertEquals(manifestMap.get(table2Path.resolve(String.format("%s-2-big-Digest.crc32", testFileConfig.getSstablePrefix(keyspace, table2)))), backupRoot.resolve(String.format("%s/2-1000000000/%s-2-big-Digest.crc32", table2, testFileConfig.getSstablePrefix(keyspace, table2))));
assertNull(manifestMap.get(table2Path.resolve(String.format("%s-3-big-Index.db", testFileConfig.getSstablePrefix(keyspace, table2)))));
}
assertNull(manifestMap.get(table1Path.resolve("manifest.json")));
assertNull(manifestMap.get(table1Path.resolve("backups")));
assertNull(manifestMap.get(table1Path.resolve("snapshots")));
}
}
Aggregations