Examples with StorageLocation com.instaclustr.esop.impl.StorageLocation used on opensource projects

Example 21 with StorageLocation

use of com.instaclustr.esop.impl.StorageLocation in project esop by instaclustr.

the class AzureBackupRestoreTest method testDownload.

@Test
public void testDownload() throws Exception {
    AzureBucketService azureBucketService = new AzureBucketService(cloudStorageAccountFactory, getBackupOperationRequest());
    Path tmp = Files.createTempDirectory("tmp");
    tmp.toFile().deleteOnExit();
    try {
        azureBucketService.create(BUCKET_NAME);
        CloudBlobClient cloudBlobClient = cloudStorageAccountFactory.build(getBackupOperationRequest()).createCloudBlobClient();
        CloudBlobContainer container = cloudBlobClient.getContainerReference(BUCKET_NAME);
        CloudBlockBlob blob1 = container.getBlockBlobReference("cluster/dc/node/manifests/snapshot-name-" + BUCKET_NAME);
        blob1.uploadText("hello");
        CloudBlockBlob blob2 = container.getBlockBlobReference("snapshot/in/dir/name-" + BUCKET_NAME);
        blob2.uploadText("hello world");
        final RestoreOperationRequest restoreOperationRequest = new RestoreOperationRequest();
        restoreOperationRequest.storageLocation = new StorageLocation("azure://" + BUCKET_NAME + "/cluster/dc/node");
        final BackupOperationRequest backupOperationRequest = new BackupOperationRequest();
        backupOperationRequest.storageLocation = new StorageLocation("azure://" + BUCKET_NAME + "/cluster/dc/node");
        final AzureRestorer azureRestorer = new AzureRestorer(cloudStorageAccountFactory, restoreOperationRequest);
        final AzureBackuper azureBackuper = new AzureBackuper(cloudStorageAccountFactory, backupOperationRequest);
        // 1
        final Path downloadedFile = azureRestorer.downloadNodeFileToDir(tmp, Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        assertTrue(Files.exists(downloadedFile));
        // 2
        final String content = azureRestorer.downloadNodeFileToString(Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        Assert.assertEquals("hello", content);
        // 3
        final String content2 = azureRestorer.downloadFileToString(Paths.get("snapshot/in/dir"), s -> s.endsWith("name-" + BUCKET_NAME));
        Assert.assertEquals("hello world", content2);
        // 4
        azureRestorer.downloadFile(tmp.resolve("some-file"), azureRestorer.objectKeyToRemoteReference(Paths.get("snapshot/in/dir/name-" + BUCKET_NAME)));
        Assert.assertTrue(Files.exists(tmp.resolve("some-file")));
        Assert.assertEquals("hello world", new String(Files.readAllBytes(tmp.resolve("some-file"))));
        // backup
        azureBackuper.uploadText("hello world", azureBackuper.objectKeyToRemoteReference(Paths.get("topology/some-file-in-here.txt")));
        String text = azureRestorer.downloadFileToString(azureBackuper.objectKeyToRemoteReference(Paths.get("topology/some-file-in-here.txt")));
        Assert.assertEquals("hello world", text);
        String topology = azureRestorer.downloadFileToString(Paths.get("topology/some-file-in"), fileName -> fileName.contains("topology/some-file-in"));
        Assert.assertEquals("hello world", topology);
    } finally {
        azureBucketService.delete(BUCKET_NAME);
        deleteDirectory(Paths.get(target("commitlog_download_dir")));
        Files.deleteIfExists(tmp.resolve("some-file"));
    }
}

Example 22 with StorageLocation

use of com.instaclustr.esop.impl.StorageLocation in project esop by instaclustr.

the class GoogleStorageBackupRestoreTest method testDownload.

@Test
public void testDownload() throws Exception {
    GCPBucketService gcpBucketService = new GCPBucketService(googleStorageFactory, getBackupOperationRequest());
    Path tmp = Files.createTempDirectory("tmp");
    tmp.toFile().deleteOnExit();
    try {
        gcpBucketService.create(BUCKET_NAME);
        Storage storage = googleStorageFactory.build(getBackupOperationRequest());
        Bucket bucket = storage.get(BUCKET_NAME);
        bucket.create("cluster/dc/node/manifests/snapshot-name-" + BUCKET_NAME, "hello".getBytes());
        bucket.create("snapshot/in/dir/name-" + BUCKET_NAME, "hello world".getBytes());
        final RestoreOperationRequest restoreOperationRequest = new RestoreOperationRequest();
        restoreOperationRequest.storageLocation = new StorageLocation("gcp://" + BUCKET_NAME + "/cluster/dc/node");
        final BackupOperationRequest backupOperationRequest = new BackupOperationRequest();
        backupOperationRequest.storageLocation = new StorageLocation("gcp://" + BUCKET_NAME + "/cluster/dc/node");
        final GCPRestorer gcpRestorer = new GCPRestorer(googleStorageFactory, restoreOperationRequest);
        final GCPBackuper gcpBackuper = new GCPBackuper(googleStorageFactory, backupOperationRequest);
        // 1
        final Path downloadedFile = gcpRestorer.downloadNodeFileToDir(tmp, Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        assertTrue(Files.exists(downloadedFile));
        // 2
        final String content = gcpRestorer.downloadNodeFileToString(Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        Assert.assertEquals("hello", content);
        // 3
        // final String content2 = gcpRestorer.downloadFileToString(Paths.get("snapshot/in/dir"), s -> s.endsWith("name-" + BUCKET_NAME));
        // Assert.assertEquals("hello world", content2);
        // 4
        gcpRestorer.downloadFile(tmp.resolve("some-file"), gcpRestorer.objectKeyToRemoteReference(Paths.get("snapshot/in/dir/name-" + BUCKET_NAME)));
        Assert.assertTrue(Files.exists(tmp.resolve("some-file")));
        Assert.assertEquals("hello world", new String(Files.readAllBytes(tmp.resolve("some-file"))));
        // backup
        gcpBackuper.uploadText("hello world", gcpBackuper.objectKeyToRemoteReference(Paths.get("topology/some-file-in-here.txt")));
        String topology = gcpRestorer.downloadFileToString(Paths.get("topology/some-file-in"), fileName -> fileName.contains("topology/some-file-in"));
        Assert.assertEquals("hello world", topology);
    } finally {
        gcpBucketService.delete(BUCKET_NAME);
        deleteDirectory(Paths.get(target("commitlog_download_dir")));
        Files.deleteIfExists(tmp.resolve("some-file"));
    }
}

Example 23 with StorageLocation

use of com.instaclustr.esop.impl.StorageLocation in project esop by instaclustr.

the class LocalBackupTest method testDownload.

@Test
public void testDownload() throws Exception {
    try {
        RestoreOperationRequest restoreOperationRequest = new RestoreOperationRequest();
        FileUtils.createDirectory(Paths.get(target("backup1") + "/cluster/test-dc/1/manifests").toAbsolutePath());
        restoreOperationRequest.storageLocation = new StorageLocation("file://" + target("backup1") + "/cluster/test-dc/1");
        Files.write(Paths.get("target/backup1/cluster/test-dc/1/manifests/snapshot-name-" + UUID.randomUUID().toString()).toAbsolutePath(), "hello".getBytes(), StandardOpenOption.CREATE_NEW);
        LocalFileRestorer localFileRestorer = new LocalFileRestorer(restoreOperationRequest);
        final Path downloadedFile = localFileRestorer.downloadNodeFileToDir(Paths.get("/tmp"), Paths.get("manifests"), s -> s.contains("snapshot-name-"));
        assertTrue(Files.exists(downloadedFile));
    } finally {
        deleteDirectory(Paths.get(target("backup1")));
        deleteDirectory(Paths.get(target("commitlog_download_dir")));
    }
}

Example 24 with StorageLocation

use of com.instaclustr.esop.impl.StorageLocation in project esop by instaclustr.

the class RemoveBackupOperation method run0.

@Override
protected void run0() throws Exception {
    assert restorerFactoryMap != null;
    assert objectMapper != null;
    request.validate(null);
    if (request.resolveNodes) {
        assert cassandraJMXService != null;
        CassandraSimpleTopologyResult simpleTopology = new CassandraSimpleTopology(cassandraJMXService).act();
        request.storageLocation = StorageLocation.update(request.storageLocation, simpleTopology.getClusterName(), simpleTopology.getDc(), simpleTopology.getHostId());
    }
    try (final StorageInteractor interactor = restorerFactoryMap.get(request.storageLocation.storageProvider).createDeletingInteractor(request)) {
        interactor.update(request.storageLocation, new LocalFileRestorer(getForLocalListing(request, request.cacheDir, request.storageLocation), objectMapper));
        for (final StorageLocation nodeLocation : getStorageLocations(interactor)) {
            logger.info("Looking for backups to delete for node {}", nodeLocation.nodePath());
            request.storageLocation = nodeLocation;
            interactor.update(nodeLocation, new LocalFileRestorer(getForLocalListing(request, request.cacheDir, request.storageLocation), objectMapper));
            final Optional<AllManifestsReport> reportOptional = getReport(interactor);
            if (!reportOptional.isPresent()) {
                logger.info("No backups found for {}", nodeLocation.nodePath());
                continue;
            }
            final AllManifestsReport report = reportOptional.get();
            logger.debug(report.toString());
            final List<ManifestReport> allBackupsToDelete = getBackupsToDelete(report);
            if (allBackupsToDelete.isEmpty()) {
                if (request.backupName != null) {
                    logger.info("There is not any {} backup to remove for node {}", request.backupName, nodeLocation);
                } else {
                    logger.info("There is not any backup to remove for node {}", nodeLocation);
                }
                continue;
            }
            logger.info("Removing backups for node {}: {}", nodeLocation.nodePath(), allBackupsToDelete.stream().map(mr -> mr.name).collect(Collectors.joining(",")));
            for (final ManifestReport mr : allBackupsToDelete) {
                interactor.delete(mr, request);
            }
        }
        if (!request.dry) {
            interactor.deleteTopology(request.backupName);
        } else {
            logger.info("Deletion of topology for {} was executed in dry mode", request.backupName);
        }
    } catch (final Exception ex) {
        logger.error("Unable to perform backup deletion! - " + ex.getMessage(), ex);
        this.addError(Error.from(ex));
    }
}