Examples with BucketService com.instaclustr.esop.impl.BucketService used on opensource projects

Example 1 with BucketService

use of com.instaclustr.esop.impl.BucketService in project esop by instaclustr.

the class InPlaceRestorationStrategy method restore.

@Override
public void restore(final Restorer restorer, final Operation<RestoreOperationRequest> operation) throws Exception {
    final RestoreOperationRequest request = operation.request;
    try {
        if (operation.request.restorationStrategyType != RestorationStrategyType.IN_PLACE) {
            throw new IllegalStateException(format("restorationStrategyType has to be of type '%s' in case you want to use %s, it is of type '%s'", RestorationStrategyType.IN_PLACE, InPlaceRestorationStrategy.class.getName(), operation.request.restorationStrategyType));
        }
        if (!request.skipBucketVerification) {
            try (final BucketService bucketService = bucketServiceFactoryMap.get(request.storageLocation.storageProvider).createBucketService(request)) {
                bucketService.checkBucket(request.storageLocation.bucket, false);
            }
        }
        if (operation.request.resolveHostIdFromTopology) {
            final NodeTopology nodeTopology = getNodeTopology(restorer, request);
            // here, nodeTopology.nodeId is uuid, not hostname
            operation.request.storageLocation = StorageLocation.updateNodeId(operation.request.storageLocation, nodeTopology.nodeId);
            restorer.update(operation.request.storageLocation, null);
            logger.info(format("Updated storage location to %s", operation.request.storageLocation));
        }
        // 2. Download the manifest & tokens, chance to error out soon before we actually pull big files if something goes wrong here
        logger.info("Retrieving manifest for snapshot: {}", request.snapshotTag);
        final Manifest manifest = RestorationUtilities.downloadManifest(operation.request, restorer, null, objectMapper);
        manifest.enrichManifestEntries();
        final DataSynchronizator synchronizator = new DataSynchronizator(manifest, request).execute();
        // here we need to categorize into what data dir entries to download will be downloaded
        // categorization will set "localFile" on manifest entry
        // we need to do it in such a way that sstables belonging together will be placed into same dir
        final ManifestEntrySSTableClassifier classifier = new ManifestEntrySSTableClassifier();
        final Map<String, List<ManifestEntry>> classified = classifier.classify(synchronizator.entriesToDownload());
        classifier.map(classified, request);
        Session<DownloadUnit> downloadSession = null;
        try {
            downloadSession = downloadTracker.submit(restorer, operation, synchronizator.entriesToDownload(), operation.request.snapshotTag, operation.request.concurrentConnections);
            downloadSession.waitUntilConsideredFinished();
            downloadTracker.cancelIfNecessary(downloadSession);
        } finally {
            downloadTracker.removeSession(downloadSession);
        }
        synchronizator.deleteUnnecessarySSTableFiles();
        synchronizator.cleanData();
        // in the future, we might implement automatic configuration of cassandra.yaml for standalone installations
        if (request.updateCassandraYaml) {
            Path fileToAppendTo;
            boolean shouldAppend = true;
            if (KubernetesHelper.isRunningInKubernetes()) {
                // Cassandra operator specific dir
                fileToAppendTo = Paths.get("/var/lib/cassandra/tokens.yaml");
            } else {
                fileToAppendTo = request.cassandraConfigDirectory.resolve("cassandra.yaml");
                if (!Files.exists(fileToAppendTo)) {
                    logger.info(String.format("File %s does not exist, not going to append to it!", fileToAppendTo));
                    shouldAppend = false;
                }
            }
            if (shouldAppend) {
                FileUtils.replaceOrAppend(fileToAppendTo, content -> content.contains("auto_bootstrap: true"), content -> !content.contains("auto_bootstrap"), "auto_bootstrap: true", "auto_bootstrap: false");
                if (FileUtils.contains(fileToAppendTo, "initial_token") && !FileUtils.contains(fileToAppendTo, "# initial_token")) {
                    logger.warn(String.format("%s file does already contain 'initial_token' property, this is unexpected and " + "backup tooling is not going to update it for you, please proceed manually, new setting should be: %s", fileToAppendTo, manifest.getInitialTokensCassandraYamlFragment()));
                } else {
                    FileUtils.appendToFile(fileToAppendTo, manifest.getInitialTokensCassandraYamlFragment());
                }
                logger.debug(String.format("Content of file %s to which necessary changes for restore were applied: ", fileToAppendTo));
                logger.debug(new String(Files.readAllBytes(fileToAppendTo)));
            }
        } else {
            logger.info("Update of cassandra.yaml was turned off by --update-cassandra-yaml=false (or not specifying that flag at all.");
            logger.info("For the successful start of a node by Cassandra operator or manually, you have to do the following:");
            logger.info("1) add tokens in Cassandra installation dir to cassandra.yaml file");
            logger.info("2) change 'auto_bootstrap: true' to 'auto_bootstrap: false' in cassandra.yaml");
        }
    } catch (final Exception ex) {
        operation.addError(Error.from(ex));
    }
}

Example 2 with BucketService

use of com.instaclustr.esop.impl.BucketService in project esop by instaclustr.

the class BaseBackupOperationCoordinator method coordinate.

@Override
public void coordinate(final Operation<BackupOperationRequest> operation) {
    final BackupOperationRequest request = operation.request;
    logger.info(request.toString());
    try {
        assert cassandraJMXService != null;
        assert backuperFactoryMap != null;
        assert bucketServiceFactoryMap != null;
        assert objectMapper != null;
        if (!request.skipBucketVerification) {
            try (final BucketService bucketService = bucketServiceFactoryMap.get(request.storageLocation.storageProvider).createBucketService(request)) {
                bucketService.checkBucket(request.storageLocation.bucket, request.createMissingBucket);
            }
        }
        final CassandraData cassandraData = CassandraData.parse(request.dataDirs.get(0));
        cassandraData.setDatabaseEntitiesFromRequest(request.entities);
        final List<String> tokens = new CassandraTokens(cassandraJMXService).act();
        logger.info("Tokens {}", tokens);
        if (!Snapshots.snapshotContainsTimestamp(operation.request.snapshotTag)) {
            if (operation.request.schemaVersion == null) {
                operation.request.schemaVersion = new CassandraSchemaVersion(cassandraJMXService).act();
            }
            operation.request.snapshotTag = resolveSnapshotTag(operation.request, System.currentTimeMillis());
        }
        logger.info("Taking snapshot with name {}", request.snapshotTag);
        new TakeSnapshotOperation(cassandraJMXService, new TakeSnapshotOperationRequest(request.entities, request.snapshotTag), cassandraVersionProvider).run0();
        Snapshots.hashSpec = hashSpec;
        final Snapshots snapshots = Snapshots.parse(request.dataDirs, request.snapshotTag);
        final Optional<Snapshot> snapshot = snapshots.get(request.snapshotTag);
        if (!snapshot.isPresent()) {
            throw new IllegalStateException(format("There is not any snapshot of tag %s", request.snapshotTag));
        }
        final Manifest manifest = Manifest.from(snapshot.get());
        manifest.setSchemaVersion(request.schemaVersion);
        manifest.setTokens(tokens);
        // manifest
        final Path localManifestPath = getLocalManifestPath(request.snapshotTag);
        Manifest.write(manifest, localManifestPath, objectMapper);
        manifest.setManifest(getManifestAsManifestEntry(localManifestPath));
        try (final Backuper backuper = backuperFactoryMap.get(request.storageLocation.storageProvider).createBackuper(request)) {
            final List<ManifestEntry> manifestEntries = manifest.getManifestEntries();
            Session<UploadUnit> uploadSession = null;
            try {
                uploadSession = uploadTracker.submit(backuper, operation, manifestEntries, request.snapshotTag, operation.request.concurrentConnections);
                uploadSession.waitUntilConsideredFinished();
                uploadTracker.cancelIfNecessary(uploadSession);
                final List<UploadUnit> failedUnits = uploadSession.getFailedUnits();
                if (!failedUnits.isEmpty()) {
                    final String message = failedUnits.stream().map(unit -> unit.getManifestEntry().objectKey.toString()).collect(Collectors.joining(","));
                    logger.error(message);
                    throw new IOException(format("Unable to upload some files successfully: %s", message));
                }
            } finally {
                uploadTracker.removeSession(uploadSession);
                uploadSession = null;
            }
            if (operation.request.uploadClusterTopology) {
                // here we will upload all topology because we do not know what restore might look like (what dc a restorer will restore against if any)
                final ClusterTopology topology = new CassandraClusterTopology(cassandraJMXService, null).act();
                ClusterTopology.upload(backuper, topology, objectMapper, operation.request.snapshotTag);
            }
        } finally {
            manifest.cleanup();
        }
    } catch (final Exception ex) {
        operation.addError(Error.from(ex));
    } finally {
        final ClearSnapshotOperation cso = new ClearSnapshotOperation(cassandraJMXService, new ClearSnapshotOperationRequest(request.snapshotTag));
        try {
            cso.run0();
        } catch (final Exception ex) {
            operation.addErrors(cso.errors);
        }
    }
}

Example 3 with BucketService

use of com.instaclustr.esop.impl.BucketService in project esop by instaclustr.

the class BackupCommitLogsOperation method run0.

@Override
protected void run0() throws Exception {
    updateStorageLocationIfNecessary();
    logger.info(request.toString());
    // generate manifest (set of object keys and source files defining the upload)
    // linked list to maintain order
    final Collection<ManifestEntry> manifestEntries = new LinkedList<>();
    try (final DirectoryStream<Path> commitLogs = getCommitLogs(request);
        final Backuper backuper = backuperFactoryMap.get(request.storageLocation.storageProvider).createCommitLogBackuper(request);
        final BucketService bucketService = bucketServiceMap.get(request.storageLocation.storageProvider).createBucketService(request)) {
        if (!request.skipBucketVerification) {
            bucketService.checkBucket(request.storageLocation.bucket, request.createMissingBucket);
        }
        for (final Path commitLog : commitLogs) {
            // Append file modified date so we have some idea of the time range this commitlog covers
            // millisecond precision, on *nix, it trims milliseconds and returns "000" instead
            // when using File.lastModified
            long commitLogLastModified = Files.getLastModifiedTime(commitLog.toFile().toPath()).toMillis();
            final Path bucketKey = CASSANDRA_COMMITLOG.resolve(commitLog.getFileName().toString() + "." + commitLogLastModified);
            manifestEntries.add(new ManifestEntry(bucketKey, commitLog, COMMIT_LOG, null));
        }
        logger.info("{} files in manifest for commitlog backup.", manifestEntries.size());
        Session<UploadUnit> uploadSession = null;
        try {
            uploadSession = uploadTracker.submit(backuper, this, manifestEntries, null, this.request.concurrentConnections);
            uploadSession.waitUntilConsideredFinished();
            uploadTracker.cancelIfNecessary(uploadSession);
        } finally {
            uploadTracker.removeSession(uploadSession);
        }
    }
}