Examples with RestoreOperationRequest com.instaclustr.esop.impl.restore.RestoreOperationRequest used on opensource projects

Example 1 with RestoreOperationRequest

use of com.instaclustr.esop.impl.restore.RestoreOperationRequest in project esop by instaclustr.

the class InPlaceRestorationStrategy method restore.

@Override
public void restore(final Restorer restorer, final Operation<RestoreOperationRequest> operation) throws Exception {
    final RestoreOperationRequest request = operation.request;
    try {
        if (operation.request.restorationStrategyType != RestorationStrategyType.IN_PLACE) {
            throw new IllegalStateException(format("restorationStrategyType has to be of type '%s' in case you want to use %s, it is of type '%s'", RestorationStrategyType.IN_PLACE, InPlaceRestorationStrategy.class.getName(), operation.request.restorationStrategyType));
        }
        if (!request.skipBucketVerification) {
            try (final BucketService bucketService = bucketServiceFactoryMap.get(request.storageLocation.storageProvider).createBucketService(request)) {
                bucketService.checkBucket(request.storageLocation.bucket, false);
            }
        }
        if (operation.request.resolveHostIdFromTopology) {
            final NodeTopology nodeTopology = getNodeTopology(restorer, request);
            // here, nodeTopology.nodeId is uuid, not hostname
            operation.request.storageLocation = StorageLocation.updateNodeId(operation.request.storageLocation, nodeTopology.nodeId);
            restorer.update(operation.request.storageLocation, null);
            logger.info(format("Updated storage location to %s", operation.request.storageLocation));
        }
        // 2. Download the manifest & tokens, chance to error out soon before we actually pull big files if something goes wrong here
        logger.info("Retrieving manifest for snapshot: {}", request.snapshotTag);
        final Manifest manifest = RestorationUtilities.downloadManifest(operation.request, restorer, null, objectMapper);
        manifest.enrichManifestEntries();
        final DataSynchronizator synchronizator = new DataSynchronizator(manifest, request).execute();
        // here we need to categorize into what data dir entries to download will be downloaded
        // categorization will set "localFile" on manifest entry
        // we need to do it in such a way that sstables belonging together will be placed into same dir
        final ManifestEntrySSTableClassifier classifier = new ManifestEntrySSTableClassifier();
        final Map<String, List<ManifestEntry>> classified = classifier.classify(synchronizator.entriesToDownload());
        classifier.map(classified, request);
        Session<DownloadUnit> downloadSession = null;
        try {
            downloadSession = downloadTracker.submit(restorer, operation, synchronizator.entriesToDownload(), operation.request.snapshotTag, operation.request.concurrentConnections);
            downloadSession.waitUntilConsideredFinished();
            downloadTracker.cancelIfNecessary(downloadSession);
        } finally {
            downloadTracker.removeSession(downloadSession);
        }
        synchronizator.deleteUnnecessarySSTableFiles();
        synchronizator.cleanData();
        // in the future, we might implement automatic configuration of cassandra.yaml for standalone installations
        if (request.updateCassandraYaml) {
            Path fileToAppendTo;
            boolean shouldAppend = true;
            if (KubernetesHelper.isRunningInKubernetes()) {
                // Cassandra operator specific dir
                fileToAppendTo = Paths.get("/var/lib/cassandra/tokens.yaml");
            } else {
                fileToAppendTo = request.cassandraConfigDirectory.resolve("cassandra.yaml");
                if (!Files.exists(fileToAppendTo)) {
                    logger.info(String.format("File %s does not exist, not going to append to it!", fileToAppendTo));
                    shouldAppend = false;
                }
            }
            if (shouldAppend) {
                FileUtils.replaceOrAppend(fileToAppendTo, content -> content.contains("auto_bootstrap: true"), content -> !content.contains("auto_bootstrap"), "auto_bootstrap: true", "auto_bootstrap: false");
                if (FileUtils.contains(fileToAppendTo, "initial_token") && !FileUtils.contains(fileToAppendTo, "# initial_token")) {
                    logger.warn(String.format("%s file does already contain 'initial_token' property, this is unexpected and " + "backup tooling is not going to update it for you, please proceed manually, new setting should be: %s", fileToAppendTo, manifest.getInitialTokensCassandraYamlFragment()));
                } else {
                    FileUtils.appendToFile(fileToAppendTo, manifest.getInitialTokensCassandraYamlFragment());
                }
                logger.debug(String.format("Content of file %s to which necessary changes for restore were applied: ", fileToAppendTo));
                logger.debug(new String(Files.readAllBytes(fileToAppendTo)));
            }
        } else {
            logger.info("Update of cassandra.yaml was turned off by --update-cassandra-yaml=false (or not specifying that flag at all.");
            logger.info("For the successful start of a node by Cassandra operator or manually, you have to do the following:");
            logger.info("1) add tokens in Cassandra installation dir to cassandra.yaml file");
            logger.info("2) change 'auto_bootstrap: true' to 'auto_bootstrap: false' in cassandra.yaml");
        }
    } catch (final Exception ex) {
        operation.addError(Error.from(ex));
    }
}

Example 2 with RestoreOperationRequest

use of com.instaclustr.esop.impl.restore.RestoreOperationRequest in project esop by instaclustr.

the class AWSS3BackupRestoreTest method testDownload.

@Test
public void testDownload() throws Exception {
    S3TransferManagerFactory factory = getTransferManagerFactory();
    S3BucketService s3BucketService = new S3BucketService(factory, getBackupOperationRequest());
    Path tmp = Files.createTempDirectory("tmp");
    tmp.toFile().deleteOnExit();
    try {
        s3BucketService.create(BUCKET_NAME);
        AmazonS3 amazonS3Client = factory.build(getBackupOperationRequest()).getAmazonS3Client();
        amazonS3Client.putObject(BUCKET_NAME, "cluster/dc/node/manifests/snapshot-name-" + BUCKET_NAME, "hello");
        amazonS3Client.putObject(BUCKET_NAME, "snapshot/in/dir/my-name-" + BUCKET_NAME, "hello world");
        amazonS3Client.listObjects(BUCKET_NAME).getObjectSummaries().forEach(summary -> logger.info(summary.getKey()));
        final RestoreOperationRequest restoreOperationRequest = new RestoreOperationRequest();
        restoreOperationRequest.storageLocation = new StorageLocation("s3://" + BUCKET_NAME + "/cluster/dc/node");
        final BackupOperationRequest backupOperationRequest = new BackupOperationRequest();
        backupOperationRequest.storageLocation = new StorageLocation("s3://" + BUCKET_NAME + "/cluster/dc/node");
        final S3Restorer s3Restorer = new S3Restorer(factory, restoreOperationRequest);
        final S3Backuper s3Backuper = new S3Backuper(factory, backupOperationRequest);
        // 1
        final Path downloadedFile = s3Restorer.downloadNodeFileToDir(tmp, Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        assertTrue(Files.exists(downloadedFile));
        // 2
        final String content = s3Restorer.downloadNodeFileToString(Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        Assert.assertEquals("hello", content);
        // 3
        final String content2 = s3Restorer.downloadFileToString(Paths.get("snapshot/in/dir"), s -> s.endsWith("my-name-" + BUCKET_NAME));
        Assert.assertEquals("hello world", content2);
        // 4
        s3Restorer.downloadFile(tmp.resolve("some-file"), s3Restorer.objectKeyToRemoteReference(Paths.get("snapshot/in/dir/my-name-" + BUCKET_NAME)));
        Assert.assertTrue(Files.exists(tmp.resolve("some-file")));
        Assert.assertEquals("hello world", new String(Files.readAllBytes(tmp.resolve("some-file"))));
        // backup
        s3Backuper.uploadText("hello world", s3Backuper.objectKeyToRemoteReference(Paths.get("topology/some-file-in-here.txt")));
        String text = s3Restorer.downloadFileToString(s3Restorer.objectKeyToRemoteReference(Paths.get("topology/some-file-in-here.txt")));
        Assert.assertEquals("hello world", text);
    } finally {
        s3BucketService.delete(BUCKET_NAME);
        deleteDirectory(Paths.get(target("commitlog_download_dir")));
        Files.deleteIfExists(tmp.resolve("some-file"));
    }
}

Example 3 with RestoreOperationRequest

use of com.instaclustr.esop.impl.restore.RestoreOperationRequest in project esop by instaclustr.

the class CephS3BackupRestoreTest method testDownload.

@Test
public void testDownload() throws Exception {
    CephS3TransferManagerFactory factory = getTransferManagerFactory();
    CephBucketService s3BucketService = new CephBucketService(factory, getBackupOperationRequest());
    Path tmp = Files.createTempDirectory("tmp");
    tmp.toFile().deleteOnExit();
    try {
        s3BucketService.create(BUCKET_NAME);
        AmazonS3 amazonS3Client = factory.build(getBackupOperationRequest()).getAmazonS3Client();
        amazonS3Client.putObject(BUCKET_NAME, "cluster/dc/node/manifests/snapshot-name-" + BUCKET_NAME, "hello");
        amazonS3Client.putObject(BUCKET_NAME, "snapshot/in/dir/my-name-" + BUCKET_NAME, "hello world");
        amazonS3Client.listObjects(BUCKET_NAME).getObjectSummaries().forEach(summary -> logger.info(summary.getKey()));
        final RestoreOperationRequest restoreOperationRequest = new RestoreOperationRequest();
        restoreOperationRequest.storageLocation = new StorageLocation("ceph://" + BUCKET_NAME + "/cluster/dc/node");
        final BackupOperationRequest backupOperationRequest = new BackupOperationRequest();
        backupOperationRequest.storageLocation = new StorageLocation("ceph://" + BUCKET_NAME + "/cluster/dc/node");
        final CephRestorer s3Restorer = new CephRestorer(factory, restoreOperationRequest);
        final CephBackuper s3Backuper = new CephBackuper(factory, backupOperationRequest);
        // 1
        final Path downloadedFile = s3Restorer.downloadNodeFileToDir(tmp, Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        assertTrue(Files.exists(downloadedFile));
        // 2
        final String content = s3Restorer.downloadNodeFileToString(Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        Assert.assertEquals("hello", content);
        // 3
        final String content2 = s3Restorer.downloadFileToString(Paths.get("snapshot/in/dir"), s -> s.endsWith("my-name-" + BUCKET_NAME));
        Assert.assertEquals("hello world", content2);
        // 4
        s3Restorer.downloadFile(tmp.resolve("some-file"), s3Restorer.objectKeyToRemoteReference(Paths.get("snapshot/in/dir/my-name-" + BUCKET_NAME)));
        Assert.assertTrue(Files.exists(tmp.resolve("some-file")));
        Assert.assertEquals("hello world", new String(Files.readAllBytes(tmp.resolve("some-file"))));
        // backup
        s3Backuper.uploadText("hello world", s3Backuper.objectKeyToRemoteReference(Paths.get("topology/some-file-in-here.txt")));
        String text = s3Restorer.downloadFileToString(s3Restorer.objectKeyToRemoteReference(Paths.get("topology/some-file-in-here.txt")));
        Assert.assertEquals("hello world", text);
    } finally {
        s3BucketService.delete(BUCKET_NAME);
        deleteDirectory(Paths.get(target("commitlog_download_dir")));
        Files.deleteIfExists(tmp.resolve("some-file"));
    }
}

Example 4 with RestoreOperationRequest

use of com.instaclustr.esop.impl.restore.RestoreOperationRequest in project esop by instaclustr.

the class AzureBackupRestoreTest method testDownload.

@Test
public void testDownload() throws Exception {
    AzureBucketService azureBucketService = new AzureBucketService(cloudStorageAccountFactory, getBackupOperationRequest());
    Path tmp = Files.createTempDirectory("tmp");
    tmp.toFile().deleteOnExit();
    try {
        azureBucketService.create(BUCKET_NAME);
        CloudBlobClient cloudBlobClient = cloudStorageAccountFactory.build(getBackupOperationRequest()).createCloudBlobClient();
        CloudBlobContainer container = cloudBlobClient.getContainerReference(BUCKET_NAME);
        CloudBlockBlob blob1 = container.getBlockBlobReference("cluster/dc/node/manifests/snapshot-name-" + BUCKET_NAME);
        blob1.uploadText("hello");
        CloudBlockBlob blob2 = container.getBlockBlobReference("snapshot/in/dir/name-" + BUCKET_NAME);
        blob2.uploadText("hello world");
        final RestoreOperationRequest restoreOperationRequest = new RestoreOperationRequest();
        restoreOperationRequest.storageLocation = new StorageLocation("azure://" + BUCKET_NAME + "/cluster/dc/node");
        final BackupOperationRequest backupOperationRequest = new BackupOperationRequest();
        backupOperationRequest.storageLocation = new StorageLocation("azure://" + BUCKET_NAME + "/cluster/dc/node");
        final AzureRestorer azureRestorer = new AzureRestorer(cloudStorageAccountFactory, restoreOperationRequest);
        final AzureBackuper azureBackuper = new AzureBackuper(cloudStorageAccountFactory, backupOperationRequest);
        // 1
        final Path downloadedFile = azureRestorer.downloadNodeFileToDir(tmp, Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        assertTrue(Files.exists(downloadedFile));
        // 2
        final String content = azureRestorer.downloadNodeFileToString(Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        Assert.assertEquals("hello", content);
        // 3
        final String content2 = azureRestorer.downloadFileToString(Paths.get("snapshot/in/dir"), s -> s.endsWith("name-" + BUCKET_NAME));
        Assert.assertEquals("hello world", content2);
        // 4
        azureRestorer.downloadFile(tmp.resolve("some-file"), azureRestorer.objectKeyToRemoteReference(Paths.get("snapshot/in/dir/name-" + BUCKET_NAME)));
        Assert.assertTrue(Files.exists(tmp.resolve("some-file")));
        Assert.assertEquals("hello world", new String(Files.readAllBytes(tmp.resolve("some-file"))));
        // backup
        azureBackuper.uploadText("hello world", azureBackuper.objectKeyToRemoteReference(Paths.get("topology/some-file-in-here.txt")));
        String text = azureRestorer.downloadFileToString(azureBackuper.objectKeyToRemoteReference(Paths.get("topology/some-file-in-here.txt")));
        Assert.assertEquals("hello world", text);
        String topology = azureRestorer.downloadFileToString(Paths.get("topology/some-file-in"), fileName -> fileName.contains("topology/some-file-in"));
        Assert.assertEquals("hello world", topology);
    } finally {
        azureBucketService.delete(BUCKET_NAME);
        deleteDirectory(Paths.get(target("commitlog_download_dir")));
        Files.deleteIfExists(tmp.resolve("some-file"));
    }
}

Example 5 with RestoreOperationRequest

use of com.instaclustr.esop.impl.restore.RestoreOperationRequest in project esop by instaclustr.

the class GoogleStorageBackupRestoreTest method testDownload.

@Test
public void testDownload() throws Exception {
    GCPBucketService gcpBucketService = new GCPBucketService(googleStorageFactory, getBackupOperationRequest());
    Path tmp = Files.createTempDirectory("tmp");
    tmp.toFile().deleteOnExit();
    try {
        gcpBucketService.create(BUCKET_NAME);
        Storage storage = googleStorageFactory.build(getBackupOperationRequest());
        Bucket bucket = storage.get(BUCKET_NAME);
        bucket.create("cluster/dc/node/manifests/snapshot-name-" + BUCKET_NAME, "hello".getBytes());
        bucket.create("snapshot/in/dir/name-" + BUCKET_NAME, "hello world".getBytes());
        final RestoreOperationRequest restoreOperationRequest = new RestoreOperationRequest();
        restoreOperationRequest.storageLocation = new StorageLocation("gcp://" + BUCKET_NAME + "/cluster/dc/node");
        final BackupOperationRequest backupOperationRequest = new BackupOperationRequest();
        backupOperationRequest.storageLocation = new StorageLocation("gcp://" + BUCKET_NAME + "/cluster/dc/node");
        final GCPRestorer gcpRestorer = new GCPRestorer(googleStorageFactory, restoreOperationRequest);
        final GCPBackuper gcpBackuper = new GCPBackuper(googleStorageFactory, backupOperationRequest);
        // 1
        final Path downloadedFile = gcpRestorer.downloadNodeFileToDir(tmp, Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        assertTrue(Files.exists(downloadedFile));
        // 2
        final String content = gcpRestorer.downloadNodeFileToString(Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        Assert.assertEquals("hello", content);
        // 3
        // final String content2 = gcpRestorer.downloadFileToString(Paths.get("snapshot/in/dir"), s -> s.endsWith("name-" + BUCKET_NAME));
        // Assert.assertEquals("hello world", content2);
        // 4
        gcpRestorer.downloadFile(tmp.resolve("some-file"), gcpRestorer.objectKeyToRemoteReference(Paths.get("snapshot/in/dir/name-" + BUCKET_NAME)));
        Assert.assertTrue(Files.exists(tmp.resolve("some-file")));
        Assert.assertEquals("hello world", new String(Files.readAllBytes(tmp.resolve("some-file"))));
        // backup
        gcpBackuper.uploadText("hello world", gcpBackuper.objectKeyToRemoteReference(Paths.get("topology/some-file-in-here.txt")));
        String topology = gcpRestorer.downloadFileToString(Paths.get("topology/some-file-in"), fileName -> fileName.contains("topology/some-file-in"));
        Assert.assertEquals("hello world", topology);
    } finally {
        gcpBucketService.delete(BUCKET_NAME);
        deleteDirectory(Paths.get(target("commitlog_download_dir")));
        Files.deleteIfExists(tmp.resolve("some-file"));
    }
}