use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.
the class LocalFileRestorer method listManifests.
@Override
public List<Manifest> listManifests() throws Exception {
assert objectMapper != null;
final Path path = Paths.get(storageLocation.rawLocation.replaceAll("file://", ""), "manifests");
if (!Files.exists(path))
return Collections.emptyList();
final List<Path> manifests = Files.list(path).sorted(new ManifestAgePathComparator()).collect(toList());
final List<Manifest> manifestsList = new ArrayList<>();
for (final Path manifest : manifests) {
final Manifest read = Manifest.read(manifest, objectMapper);
read.setManifest(new ManifestEntry(Paths.get("manifests", manifest.getFileName().toString()), manifest, Type.FILE, null));
manifestsList.add(read);
}
return manifestsList;
}
use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.
the class BaseBackupOperationCoordinator method coordinate.
@Override
public void coordinate(final Operation<BackupOperationRequest> operation) {
final BackupOperationRequest request = operation.request;
logger.info(request.toString());
try {
assert cassandraJMXService != null;
assert backuperFactoryMap != null;
assert bucketServiceFactoryMap != null;
assert objectMapper != null;
if (!request.skipBucketVerification) {
try (final BucketService bucketService = bucketServiceFactoryMap.get(request.storageLocation.storageProvider).createBucketService(request)) {
bucketService.checkBucket(request.storageLocation.bucket, request.createMissingBucket);
}
}
final CassandraData cassandraData = CassandraData.parse(request.dataDirs.get(0));
cassandraData.setDatabaseEntitiesFromRequest(request.entities);
final List<String> tokens = new CassandraTokens(cassandraJMXService).act();
logger.info("Tokens {}", tokens);
if (!Snapshots.snapshotContainsTimestamp(operation.request.snapshotTag)) {
if (operation.request.schemaVersion == null) {
operation.request.schemaVersion = new CassandraSchemaVersion(cassandraJMXService).act();
}
operation.request.snapshotTag = resolveSnapshotTag(operation.request, System.currentTimeMillis());
}
logger.info("Taking snapshot with name {}", request.snapshotTag);
new TakeSnapshotOperation(cassandraJMXService, new TakeSnapshotOperationRequest(request.entities, request.snapshotTag), cassandraVersionProvider).run0();
Snapshots.hashSpec = hashSpec;
final Snapshots snapshots = Snapshots.parse(request.dataDirs, request.snapshotTag);
final Optional<Snapshot> snapshot = snapshots.get(request.snapshotTag);
if (!snapshot.isPresent()) {
throw new IllegalStateException(format("There is not any snapshot of tag %s", request.snapshotTag));
}
final Manifest manifest = Manifest.from(snapshot.get());
manifest.setSchemaVersion(request.schemaVersion);
manifest.setTokens(tokens);
// manifest
final Path localManifestPath = getLocalManifestPath(request.snapshotTag);
Manifest.write(manifest, localManifestPath, objectMapper);
manifest.setManifest(getManifestAsManifestEntry(localManifestPath));
try (final Backuper backuper = backuperFactoryMap.get(request.storageLocation.storageProvider).createBackuper(request)) {
final List<ManifestEntry> manifestEntries = manifest.getManifestEntries();
Session<UploadUnit> uploadSession = null;
try {
uploadSession = uploadTracker.submit(backuper, operation, manifestEntries, request.snapshotTag, operation.request.concurrentConnections);
uploadSession.waitUntilConsideredFinished();
uploadTracker.cancelIfNecessary(uploadSession);
final List<UploadUnit> failedUnits = uploadSession.getFailedUnits();
if (!failedUnits.isEmpty()) {
final String message = failedUnits.stream().map(unit -> unit.getManifestEntry().objectKey.toString()).collect(Collectors.joining(","));
logger.error(message);
throw new IOException(format("Unable to upload some files successfully: %s", message));
}
} finally {
uploadTracker.removeSession(uploadSession);
uploadSession = null;
}
if (operation.request.uploadClusterTopology) {
// here we will upload all topology because we do not know what restore might look like (what dc a restorer will restore against if any)
final ClusterTopology topology = new CassandraClusterTopology(cassandraJMXService, null).act();
ClusterTopology.upload(backuper, topology, objectMapper, operation.request.snapshotTag);
}
} finally {
manifest.cleanup();
}
} catch (final Exception ex) {
operation.addError(Error.from(ex));
} finally {
final ClearSnapshotOperation cso = new ClearSnapshotOperation(cassandraJMXService, new ClearSnapshotOperationRequest(request.snapshotTag));
try {
cso.run0();
} catch (final Exception ex) {
operation.addErrors(cso.errors);
}
}
}
use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.
the class RestoreCommitLogsOperation method downloadCommitLogs.
private void downloadCommitLogs(final Restorer restorer) throws Exception {
final RemoteObjectReference remoteObjectReference = restorer.objectKeyToNodeAwareRemoteReference(Paths.get("commitlog"));
final Pattern commitlogPattern = Pattern.compile(".*(CommitLog-\\d+-\\d+\\.log)\\.(\\d+)");
final Set<ManifestEntry> parsedCommitlogList = new HashSet<>();
logger.info("Commencing processing of commit log listing");
final AtomicReference<ManifestEntry> overhangingManifestEntry = new AtomicReference<>();
final AtomicLong overhangingTimestamp = new AtomicLong(Long.MAX_VALUE);
restorer.consumeFiles(remoteObjectReference, commitlogFile -> {
final Matcher matcherCommitlog = commitlogPattern.matcher(commitlogFile.getObjectKey().toString());
if (matcherCommitlog.matches()) {
final long commitlogTimestamp = Long.parseLong(matcherCommitlog.group(2));
if (commitlogTimestamp >= request.timestampStart && commitlogTimestamp <= request.timestampEnd) {
parsedCommitlogList.add(new ManifestEntry(commitlogFile.getObjectKey(), request.commitlogDownloadDir.resolve(matcherCommitlog.group(1)), COMMIT_LOG, 0, null, null));
} else if (commitlogTimestamp > request.timestampEnd && commitlogTimestamp < overhangingTimestamp.get()) {
// Make sure we also catch the first commitlog that goes past the end of the timestamp
overhangingTimestamp.set(commitlogTimestamp);
overhangingManifestEntry.set(new ManifestEntry(commitlogFile.getObjectKey(), request.commitlogDownloadDir.resolve(matcherCommitlog.group(1)), COMMIT_LOG, 0, null, null));
}
}
});
if (overhangingManifestEntry.get() != null) {
parsedCommitlogList.add(overhangingManifestEntry.get());
}
logger.info("Found {} commit logs to download", parsedCommitlogList.size());
if (parsedCommitlogList.size() == 0) {
return;
}
Session<DownloadUnit> downloadSession = null;
try {
downloadSession = downloadTracker.submit(restorer, this, parsedCommitlogList, null, this.request.concurrentConnections);
downloadSession.waitUntilConsideredFinished();
downloadTracker.cancelIfNecessary(downloadSession);
} finally {
downloadTracker.removeSession(downloadSession);
}
}
use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.
the class ManifestComponentsTest method testManifestFilterNoRestoreSystemKeyspaceYesNewCluster.
@Test
public void testManifestFilterNoRestoreSystemKeyspaceYesNewCluster() throws Exception {
Manifest manifest = parseManifest();
manifest.enrichManifestEntries();
List<ManifestEntry> manifestFiles = manifest.getManifestFiles(new DatabaseEntities(), // restoreSystemKeyspace
false, // restoreSystemAuth
false, // newCluster
true, // withSchemas
false);
Assert.assertTrue(manifestFiles.stream().allMatch(entry -> {
if (entry.keyspaceTable.keyspace.equals("system")) {
return entry.keyspaceTable.table.startsWith("schema_");
} else if (entry.keyspaceTable.keyspace.equals("system_schema")) {
return true;
} else
return !KeyspaceTable.isSystemKeyspace(entry.keyspaceTable.keyspace);
}));
}
use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.
the class ManifestComponentsTest method testManifestFilterYesRestoreSystemKeyspaceYesNewCluster.
@Test
public void testManifestFilterYesRestoreSystemKeyspaceYesNewCluster() throws Exception {
Manifest manifest = parseManifest();
manifest.enrichManifestEntries();
List<ManifestEntry> manifestFiles = manifest.getManifestFiles(new DatabaseEntities(), // restoreSystemKeyspace
true, // restoreSystemAuth
false, // newCluster
true, // withSchemas
false);
Assert.assertEquals(manifestFiles.size(), 304);
}
Aggregations