Examples with BackupOperationRequest com.instaclustr.esop.impl.backup.BackupOperationRequest used on opensource projects

Example 1 with BackupOperationRequest

use of com.instaclustr.esop.impl.backup.BackupOperationRequest in project esop by instaclustr.

the class BaseBackupOperationCoordinator method coordinate.

@Override
public void coordinate(final Operation<BackupOperationRequest> operation) {
    final BackupOperationRequest request = operation.request;
    logger.info(request.toString());
    try {
        assert cassandraJMXService != null;
        assert backuperFactoryMap != null;
        assert bucketServiceFactoryMap != null;
        assert objectMapper != null;
        if (!request.skipBucketVerification) {
            try (final BucketService bucketService = bucketServiceFactoryMap.get(request.storageLocation.storageProvider).createBucketService(request)) {
                bucketService.checkBucket(request.storageLocation.bucket, request.createMissingBucket);
            }
        }
        final CassandraData cassandraData = CassandraData.parse(request.dataDirs.get(0));
        cassandraData.setDatabaseEntitiesFromRequest(request.entities);
        final List<String> tokens = new CassandraTokens(cassandraJMXService).act();
        logger.info("Tokens {}", tokens);
        if (!Snapshots.snapshotContainsTimestamp(operation.request.snapshotTag)) {
            if (operation.request.schemaVersion == null) {
                operation.request.schemaVersion = new CassandraSchemaVersion(cassandraJMXService).act();
            }
            operation.request.snapshotTag = resolveSnapshotTag(operation.request, System.currentTimeMillis());
        }
        logger.info("Taking snapshot with name {}", request.snapshotTag);
        new TakeSnapshotOperation(cassandraJMXService, new TakeSnapshotOperationRequest(request.entities, request.snapshotTag), cassandraVersionProvider).run0();
        Snapshots.hashSpec = hashSpec;
        final Snapshots snapshots = Snapshots.parse(request.dataDirs, request.snapshotTag);
        final Optional<Snapshot> snapshot = snapshots.get(request.snapshotTag);
        if (!snapshot.isPresent()) {
            throw new IllegalStateException(format("There is not any snapshot of tag %s", request.snapshotTag));
        }
        final Manifest manifest = Manifest.from(snapshot.get());
        manifest.setSchemaVersion(request.schemaVersion);
        manifest.setTokens(tokens);
        // manifest
        final Path localManifestPath = getLocalManifestPath(request.snapshotTag);
        Manifest.write(manifest, localManifestPath, objectMapper);
        manifest.setManifest(getManifestAsManifestEntry(localManifestPath));
        try (final Backuper backuper = backuperFactoryMap.get(request.storageLocation.storageProvider).createBackuper(request)) {
            final List<ManifestEntry> manifestEntries = manifest.getManifestEntries();
            Session<UploadUnit> uploadSession = null;
            try {
                uploadSession = uploadTracker.submit(backuper, operation, manifestEntries, request.snapshotTag, operation.request.concurrentConnections);
                uploadSession.waitUntilConsideredFinished();
                uploadTracker.cancelIfNecessary(uploadSession);
                final List<UploadUnit> failedUnits = uploadSession.getFailedUnits();
                if (!failedUnits.isEmpty()) {
                    final String message = failedUnits.stream().map(unit -> unit.getManifestEntry().objectKey.toString()).collect(Collectors.joining(","));
                    logger.error(message);
                    throw new IOException(format("Unable to upload some files successfully: %s", message));
                }
            } finally {
                uploadTracker.removeSession(uploadSession);
                uploadSession = null;
            }
            if (operation.request.uploadClusterTopology) {
                // here we will upload all topology because we do not know what restore might look like (what dc a restorer will restore against if any)
                final ClusterTopology topology = new CassandraClusterTopology(cassandraJMXService, null).act();
                ClusterTopology.upload(backuper, topology, objectMapper, operation.request.snapshotTag);
            }
        } finally {
            manifest.cleanup();
        }
    } catch (final Exception ex) {
        operation.addError(Error.from(ex));
    } finally {
        final ClearSnapshotOperation cso = new ClearSnapshotOperation(cassandraJMXService, new ClearSnapshotOperationRequest(request.snapshotTag));
        try {
            cso.run0();
        } catch (final Exception ex) {
            operation.addErrors(cso.errors);
        }
    }
}

Example 2 with BackupOperationRequest

use of com.instaclustr.esop.impl.backup.BackupOperationRequest in project esop by instaclustr.

the class AWSS3BackupRestoreTest method testDownload.

@Test
public void testDownload() throws Exception {
    S3TransferManagerFactory factory = getTransferManagerFactory();
    S3BucketService s3BucketService = new S3BucketService(factory, getBackupOperationRequest());
    Path tmp = Files.createTempDirectory("tmp");
    tmp.toFile().deleteOnExit();
    try {
        s3BucketService.create(BUCKET_NAME);
        AmazonS3 amazonS3Client = factory.build(getBackupOperationRequest()).getAmazonS3Client();
        amazonS3Client.putObject(BUCKET_NAME, "cluster/dc/node/manifests/snapshot-name-" + BUCKET_NAME, "hello");
        amazonS3Client.putObject(BUCKET_NAME, "snapshot/in/dir/my-name-" + BUCKET_NAME, "hello world");
        amazonS3Client.listObjects(BUCKET_NAME).getObjectSummaries().forEach(summary -> logger.info(summary.getKey()));
        final RestoreOperationRequest restoreOperationRequest = new RestoreOperationRequest();
        restoreOperationRequest.storageLocation = new StorageLocation("s3://" + BUCKET_NAME + "/cluster/dc/node");
        final BackupOperationRequest backupOperationRequest = new BackupOperationRequest();
        backupOperationRequest.storageLocation = new StorageLocation("s3://" + BUCKET_NAME + "/cluster/dc/node");
        final S3Restorer s3Restorer = new S3Restorer(factory, restoreOperationRequest);
        final S3Backuper s3Backuper = new S3Backuper(factory, backupOperationRequest);
        // 1
        final Path downloadedFile = s3Restorer.downloadNodeFileToDir(tmp, Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        assertTrue(Files.exists(downloadedFile));
        // 2
        final String content = s3Restorer.downloadNodeFileToString(Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        Assert.assertEquals("hello", content);
        // 3
        final String content2 = s3Restorer.downloadFileToString(Paths.get("snapshot/in/dir"), s -> s.endsWith("my-name-" + BUCKET_NAME));
        Assert.assertEquals("hello world", content2);
        // 4
        s3Restorer.downloadFile(tmp.resolve("some-file"), s3Restorer.objectKeyToRemoteReference(Paths.get("snapshot/in/dir/my-name-" + BUCKET_NAME)));
        Assert.assertTrue(Files.exists(tmp.resolve("some-file")));
        Assert.assertEquals("hello world", new String(Files.readAllBytes(tmp.resolve("some-file"))));
        // backup
        s3Backuper.uploadText("hello world", s3Backuper.objectKeyToRemoteReference(Paths.get("topology/some-file-in-here.txt")));
        String text = s3Restorer.downloadFileToString(s3Restorer.objectKeyToRemoteReference(Paths.get("topology/some-file-in-here.txt")));
        Assert.assertEquals("hello world", text);
    } finally {
        s3BucketService.delete(BUCKET_NAME);
        deleteDirectory(Paths.get(target("commitlog_download_dir")));
        Files.deleteIfExists(tmp.resolve("some-file"));
    }
}

Example 3 with BackupOperationRequest

use of com.instaclustr.esop.impl.backup.BackupOperationRequest in project esop by instaclustr.

the class CephS3BackupRestoreTest method testDownload.

@Test
public void testDownload() throws Exception {
    CephS3TransferManagerFactory factory = getTransferManagerFactory();
    CephBucketService s3BucketService = new CephBucketService(factory, getBackupOperationRequest());
    Path tmp = Files.createTempDirectory("tmp");
    tmp.toFile().deleteOnExit();
    try {
        s3BucketService.create(BUCKET_NAME);
        AmazonS3 amazonS3Client = factory.build(getBackupOperationRequest()).getAmazonS3Client();
        amazonS3Client.putObject(BUCKET_NAME, "cluster/dc/node/manifests/snapshot-name-" + BUCKET_NAME, "hello");
        amazonS3Client.putObject(BUCKET_NAME, "snapshot/in/dir/my-name-" + BUCKET_NAME, "hello world");
        amazonS3Client.listObjects(BUCKET_NAME).getObjectSummaries().forEach(summary -> logger.info(summary.getKey()));
        final RestoreOperationRequest restoreOperationRequest = new RestoreOperationRequest();
        restoreOperationRequest.storageLocation = new StorageLocation("ceph://" + BUCKET_NAME + "/cluster/dc/node");
        final BackupOperationRequest backupOperationRequest = new BackupOperationRequest();
        backupOperationRequest.storageLocation = new StorageLocation("ceph://" + BUCKET_NAME + "/cluster/dc/node");
        final CephRestorer s3Restorer = new CephRestorer(factory, restoreOperationRequest);
        final CephBackuper s3Backuper = new CephBackuper(factory, backupOperationRequest);
        // 1
        final Path downloadedFile = s3Restorer.downloadNodeFileToDir(tmp, Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        assertTrue(Files.exists(downloadedFile));
        // 2
        final String content = s3Restorer.downloadNodeFileToString(Paths.get("manifests"), s -> s.contains("manifests/snapshot-name"));
        Assert.assertEquals("hello", content);
        // 3
        final String content2 = s3Restorer.downloadFileToString(Paths.get("snapshot/in/dir"), s -> s.endsWith("my-name-" + BUCKET_NAME));
        Assert.assertEquals("hello world", content2);
        // 4
        s3Restorer.downloadFile(tmp.resolve("some-file"), s3Restorer.objectKeyToRemoteReference(Paths.get("snapshot/in/dir/my-name-" + BUCKET_NAME)));
        Assert.assertTrue(Files.exists(tmp.resolve("some-file")));
        Assert.assertEquals("hello world", new String(Files.readAllBytes(tmp.resolve("some-file"))));
        // backup
        s3Backuper.uploadText("hello world", s3Backuper.objectKeyToRemoteReference(Paths.get("topology/some-file-in-here.txt")));
        String text = s3Restorer.downloadFileToString(s3Restorer.objectKeyToRemoteReference(Paths.get("topology/some-file-in-here.txt")));
        Assert.assertEquals("hello world", text);
    } finally {
        s3BucketService.delete(BUCKET_NAME);
        deleteDirectory(Paths.get(target("commitlog_download_dir")));
        Files.deleteIfExists(tmp.resolve("some-file"));
    }
}

Example 4 with BackupOperationRequest

use of com.instaclustr.esop.impl.backup.BackupOperationRequest in project esop by instaclustr.

the class KubernetesCephS3BackupRestoreTest method getBackupOperationRequest.

@Override
protected BackupOperationRequest getBackupOperationRequest() {
    final BackupOperationRequest backupOperationRequest = new BackupOperationRequest();
    backupOperationRequest.k8sSecretName = SIDECAR_SECRET_NAME;
    return backupOperationRequest;
}

Example 5 with BackupOperationRequest

use of com.instaclustr.esop.impl.backup.BackupOperationRequest in project esop by instaclustr.

the class LocalBackupTest method testUploadTracker.

@Test
public void testUploadTracker() throws Exception {
    final String snapshotName = UUID.randomUUID().toString();
    final String snapshotName2 = UUID.randomUUID().toString();
    List<Path> dataDirs = Arrays.asList(cassandraDir.toAbsolutePath().resolve("data").resolve("data"), cassandraDir.toAbsolutePath().resolve("data").resolve("data2"), cassandraDir.toAbsolutePath().resolve("data").resolve("data3"));
    final BackupOperationRequest backupOperationRequest = getBackupOperationRequestForTracker(snapshotName, "test,test2", dataDirs);
    final BackupOperationRequest backupOperationRequest2 = getBackupOperationRequestForTracker(snapshotName2, "test", dataDirs);
    UploadTracker uploadTracker = null;
    Cassandra cassandra = null;
    try {
        cassandra = getCassandra(cassandraDir, CASSANDRA_VERSION);
        cassandra.start();
        try (CqlSession session = CqlSession.builder().build()) {
            assertEquals(populateDatabase(session).size(), NUMBER_OF_INSERTED_ROWS);
        }
        final AtomicBoolean wait = new AtomicBoolean(true);
        final ListeningExecutorService finisher = new Executors.FixedTasksExecutorSupplier().get(10);
        uploadTracker = new UploadTracker(finisher, operationsService, new HashSpec()) {

            // override for testing purposes
            @Override
            public UploadUnit constructUnitToSubmit(final Backuper backuper, final ManifestEntry manifestEntry, final AtomicBoolean shouldCancel, final String snapshotTag, final HashSpec hashSpec) {
                return new TestingUploadUnit(wait, backuper, manifestEntry, shouldCancel, snapshotTag, hashSpec);
            }
        };
        final LocalFileBackuper backuper = new LocalFileBackuper(backupOperationRequest);
        new TakeSnapshotOperation(jmxService, new TakeSnapshotOperationRequest(backupOperationRequest.entities, backupOperationRequest.snapshotTag), cassandraVersionProvider).run();
        new TakeSnapshotOperation(jmxService, new TakeSnapshotOperationRequest(backupOperationRequest2.entities, backupOperationRequest2.snapshotTag), cassandraVersionProvider).run();
        final Snapshots snapshots = Snapshots.parse(dataDirs);
        final Optional<Snapshot> snapshot = snapshots.get(backupOperationRequest.snapshotTag);
        final Optional<Snapshot> snapshot2 = snapshots.get(backupOperationRequest2.snapshotTag);
        assert snapshot.isPresent();
        assert snapshot2.isPresent();
        Set<String> providers = Stream.of("file").collect(Collectors.toSet());
        final BackupOperation backupOperation = new BackupOperation(operationCoordinator, providers, backupOperationRequest);
        final BackupOperation backupOperation2 = new BackupOperation(operationCoordinator, providers, backupOperationRequest2);
        final List<ManifestEntry> manifestEntries = Manifest.from(snapshot.get()).getManifestEntries();
        final List<ManifestEntry> manifestEntries2 = Manifest.from(snapshot2.get()).getManifestEntries();
        Session<UploadUnit> session = uploadTracker.submit(backuper, backupOperation, manifestEntries, backupOperation.request.snapshotTag, backupOperation.request.concurrentConnections);
        final int submittedUnits1 = uploadTracker.submittedUnits.intValue();
        Assert.assertEquals(manifestEntries.size(), submittedUnits1);
        final Session<UploadUnit> session2 = uploadTracker.submit(backuper, backupOperation2, manifestEntries2, backupOperation.request.snapshotTag, backupOperation.request.concurrentConnections);
        final int submittedUnits2 = uploadTracker.submittedUnits.intValue();
        // even we submitted the second session, it does not change the number of units because session2
        // wants to upload "test" but it is already going to be uploaded by session1
        // we have effectively submitted only what should be submitted, no duplicates
        // so it is as if "test" from session2 was not submitted at all
        Assert.assertEquals(submittedUnits1, submittedUnits2);
        Assert.assertEquals(manifestEntries.size(), uploadTracker.submittedUnits.intValue());
        // however we have submitted two sessions in total
        Assert.assertEquals(2, uploadTracker.submittedSessions.intValue());
        // lets upload it now
        wait.set(false);
        session.waitUntilConsideredFinished();
        session2.waitUntilConsideredFinished();
        Assert.assertTrue(session.isConsideredFinished());
        Assert.assertTrue(session.isSuccessful());
        Assert.assertTrue(session.getFailedUnits().isEmpty());
        Assert.assertEquals(uploadTracker.submittedUnits.intValue(), session.getUnits().size());
        Assert.assertTrue(session2.isConsideredFinished());
        Assert.assertTrue(session2.isSuccessful());
        Assert.assertTrue(session2.getFailedUnits().isEmpty());
        Assert.assertTrue(submittedUnits2 > session2.getUnits().size());
        for (final UploadUnit uploadUnit : session2.getUnits()) {
            Assert.assertTrue(session.getUnits().contains(uploadUnit));
        }
        Assert.assertTrue(uploadTracker.getUnits().isEmpty());
        uploadTracker.removeSession(session);
        uploadTracker.removeSession(session2);
        Assert.assertTrue(session.getUnits().isEmpty());
        Assert.assertTrue(session2.getUnits().isEmpty());
    } catch (final Exception ex) {
        ex.printStackTrace();
        throw ex;
    } finally {
        new ClearSnapshotOperation(jmxService, new ClearSnapshotOperationRequest(backupOperationRequest.snapshotTag)).run();
        if (cassandra != null) {
            cassandra.stop();
        }
        uploadTracker.stopAsync();
        uploadTracker.awaitTerminated(1, MINUTES);
        uploadTracker.stopAsync();
        uploadTracker.awaitTerminated(1, MINUTES);
        FileUtils.deleteDirectory(Paths.get(target(backupOperationRequest.storageLocation.bucket)));
    }
}