Examples with DownloadUnit com.instaclustr.esop.impl.restore.DownloadTracker.DownloadUnit used on opensource projects

Example 1 with DownloadUnit

use of com.instaclustr.esop.impl.restore.DownloadTracker.DownloadUnit in project esop by instaclustr.

the class InPlaceRestorationStrategy method restore.

@Override
public void restore(final Restorer restorer, final Operation<RestoreOperationRequest> operation) throws Exception {
    final RestoreOperationRequest request = operation.request;
    try {
        if (operation.request.restorationStrategyType != RestorationStrategyType.IN_PLACE) {
            throw new IllegalStateException(format("restorationStrategyType has to be of type '%s' in case you want to use %s, it is of type '%s'", RestorationStrategyType.IN_PLACE, InPlaceRestorationStrategy.class.getName(), operation.request.restorationStrategyType));
        }
        if (!request.skipBucketVerification) {
            try (final BucketService bucketService = bucketServiceFactoryMap.get(request.storageLocation.storageProvider).createBucketService(request)) {
                bucketService.checkBucket(request.storageLocation.bucket, false);
            }
        }
        if (operation.request.resolveHostIdFromTopology) {
            final NodeTopology nodeTopology = getNodeTopology(restorer, request);
            // here, nodeTopology.nodeId is uuid, not hostname
            operation.request.storageLocation = StorageLocation.updateNodeId(operation.request.storageLocation, nodeTopology.nodeId);
            restorer.update(operation.request.storageLocation, null);
            logger.info(format("Updated storage location to %s", operation.request.storageLocation));
        }
        // 2. Download the manifest & tokens, chance to error out soon before we actually pull big files if something goes wrong here
        logger.info("Retrieving manifest for snapshot: {}", request.snapshotTag);
        final Manifest manifest = RestorationUtilities.downloadManifest(operation.request, restorer, null, objectMapper);
        manifest.enrichManifestEntries();
        final DataSynchronizator synchronizator = new DataSynchronizator(manifest, request).execute();
        // here we need to categorize into what data dir entries to download will be downloaded
        // categorization will set "localFile" on manifest entry
        // we need to do it in such a way that sstables belonging together will be placed into same dir
        final ManifestEntrySSTableClassifier classifier = new ManifestEntrySSTableClassifier();
        final Map<String, List<ManifestEntry>> classified = classifier.classify(synchronizator.entriesToDownload());
        classifier.map(classified, request);
        Session<DownloadUnit> downloadSession = null;
        try {
            downloadSession = downloadTracker.submit(restorer, operation, synchronizator.entriesToDownload(), operation.request.snapshotTag, operation.request.concurrentConnections);
            downloadSession.waitUntilConsideredFinished();
            downloadTracker.cancelIfNecessary(downloadSession);
        } finally {
            downloadTracker.removeSession(downloadSession);
        }
        synchronizator.deleteUnnecessarySSTableFiles();
        synchronizator.cleanData();
        // in the future, we might implement automatic configuration of cassandra.yaml for standalone installations
        if (request.updateCassandraYaml) {
            Path fileToAppendTo;
            boolean shouldAppend = true;
            if (KubernetesHelper.isRunningInKubernetes()) {
                // Cassandra operator specific dir
                fileToAppendTo = Paths.get("/var/lib/cassandra/tokens.yaml");
            } else {
                fileToAppendTo = request.cassandraConfigDirectory.resolve("cassandra.yaml");
                if (!Files.exists(fileToAppendTo)) {
                    logger.info(String.format("File %s does not exist, not going to append to it!", fileToAppendTo));
                    shouldAppend = false;
                }
            }
            if (shouldAppend) {
                FileUtils.replaceOrAppend(fileToAppendTo, content -> content.contains("auto_bootstrap: true"), content -> !content.contains("auto_bootstrap"), "auto_bootstrap: true", "auto_bootstrap: false");
                if (FileUtils.contains(fileToAppendTo, "initial_token") && !FileUtils.contains(fileToAppendTo, "# initial_token")) {
                    logger.warn(String.format("%s file does already contain 'initial_token' property, this is unexpected and " + "backup tooling is not going to update it for you, please proceed manually, new setting should be: %s", fileToAppendTo, manifest.getInitialTokensCassandraYamlFragment()));
                } else {
                    FileUtils.appendToFile(fileToAppendTo, manifest.getInitialTokensCassandraYamlFragment());
                }
                logger.debug(String.format("Content of file %s to which necessary changes for restore were applied: ", fileToAppendTo));
                logger.debug(new String(Files.readAllBytes(fileToAppendTo)));
            }
        } else {
            logger.info("Update of cassandra.yaml was turned off by --update-cassandra-yaml=false (or not specifying that flag at all.");
            logger.info("For the successful start of a node by Cassandra operator or manually, you have to do the following:");
            logger.info("1) add tokens in Cassandra installation dir to cassandra.yaml file");
            logger.info("2) change 'auto_bootstrap: true' to 'auto_bootstrap: false' in cassandra.yaml");
        }
    } catch (final Exception ex) {
        operation.addError(Error.from(ex));
    }
}

Example 2 with DownloadUnit

use of com.instaclustr.esop.impl.restore.DownloadTracker.DownloadUnit in project esop by instaclustr.

the class RestoreCommitLogsOperation method downloadCommitLogs.

private void downloadCommitLogs(final Restorer restorer) throws Exception {
    final RemoteObjectReference remoteObjectReference = restorer.objectKeyToNodeAwareRemoteReference(Paths.get("commitlog"));
    final Pattern commitlogPattern = Pattern.compile(".*(CommitLog-\\d+-\\d+\\.log)\\.(\\d+)");
    final Set<ManifestEntry> parsedCommitlogList = new HashSet<>();
    logger.info("Commencing processing of commit log listing");
    final AtomicReference<ManifestEntry> overhangingManifestEntry = new AtomicReference<>();
    final AtomicLong overhangingTimestamp = new AtomicLong(Long.MAX_VALUE);
    restorer.consumeFiles(remoteObjectReference, commitlogFile -> {
        final Matcher matcherCommitlog = commitlogPattern.matcher(commitlogFile.getObjectKey().toString());
        if (matcherCommitlog.matches()) {
            final long commitlogTimestamp = Long.parseLong(matcherCommitlog.group(2));
            if (commitlogTimestamp >= request.timestampStart && commitlogTimestamp <= request.timestampEnd) {
                parsedCommitlogList.add(new ManifestEntry(commitlogFile.getObjectKey(), request.commitlogDownloadDir.resolve(matcherCommitlog.group(1)), COMMIT_LOG, 0, null, null));
            } else if (commitlogTimestamp > request.timestampEnd && commitlogTimestamp < overhangingTimestamp.get()) {
                // Make sure we also catch the first commitlog that goes past the end of the timestamp
                overhangingTimestamp.set(commitlogTimestamp);
                overhangingManifestEntry.set(new ManifestEntry(commitlogFile.getObjectKey(), request.commitlogDownloadDir.resolve(matcherCommitlog.group(1)), COMMIT_LOG, 0, null, null));
            }
        }
    });
    if (overhangingManifestEntry.get() != null) {
        parsedCommitlogList.add(overhangingManifestEntry.get());
    }
    logger.info("Found {} commit logs to download", parsedCommitlogList.size());
    if (parsedCommitlogList.size() == 0) {
        return;
    }
    Session<DownloadUnit> downloadSession = null;
    try {
        downloadSession = downloadTracker.submit(restorer, this, parsedCommitlogList, null, this.request.concurrentConnections);
        downloadSession.waitUntilConsideredFinished();
        downloadTracker.cancelIfNecessary(downloadSession);
    } finally {
        downloadTracker.removeSession(downloadSession);
    }
}