Examples with CassandraTokens com.instaclustr.esop.impl.interaction.CassandraTokens used on opensource projects

Example 1 with CassandraTokens

use of com.instaclustr.esop.impl.interaction.CassandraTokens in project esop by instaclustr.

the class BaseBackupOperationCoordinator method coordinate.

@Override
public void coordinate(final Operation<BackupOperationRequest> operation) {
    final BackupOperationRequest request = operation.request;
    logger.info(request.toString());
    try {
        assert cassandraJMXService != null;
        assert backuperFactoryMap != null;
        assert bucketServiceFactoryMap != null;
        assert objectMapper != null;
        if (!request.skipBucketVerification) {
            try (final BucketService bucketService = bucketServiceFactoryMap.get(request.storageLocation.storageProvider).createBucketService(request)) {
                bucketService.checkBucket(request.storageLocation.bucket, request.createMissingBucket);
            }
        }
        final CassandraData cassandraData = CassandraData.parse(request.dataDirs.get(0));
        cassandraData.setDatabaseEntitiesFromRequest(request.entities);
        final List<String> tokens = new CassandraTokens(cassandraJMXService).act();
        logger.info("Tokens {}", tokens);
        if (!Snapshots.snapshotContainsTimestamp(operation.request.snapshotTag)) {
            if (operation.request.schemaVersion == null) {
                operation.request.schemaVersion = new CassandraSchemaVersion(cassandraJMXService).act();
            }
            operation.request.snapshotTag = resolveSnapshotTag(operation.request, System.currentTimeMillis());
        }
        logger.info("Taking snapshot with name {}", request.snapshotTag);
        new TakeSnapshotOperation(cassandraJMXService, new TakeSnapshotOperationRequest(request.entities, request.snapshotTag), cassandraVersionProvider).run0();
        Snapshots.hashSpec = hashSpec;
        final Snapshots snapshots = Snapshots.parse(request.dataDirs, request.snapshotTag);
        final Optional<Snapshot> snapshot = snapshots.get(request.snapshotTag);
        if (!snapshot.isPresent()) {
            throw new IllegalStateException(format("There is not any snapshot of tag %s", request.snapshotTag));
        }
        final Manifest manifest = Manifest.from(snapshot.get());
        manifest.setSchemaVersion(request.schemaVersion);
        manifest.setTokens(tokens);
        // manifest
        final Path localManifestPath = getLocalManifestPath(request.snapshotTag);
        Manifest.write(manifest, localManifestPath, objectMapper);
        manifest.setManifest(getManifestAsManifestEntry(localManifestPath));
        try (final Backuper backuper = backuperFactoryMap.get(request.storageLocation.storageProvider).createBackuper(request)) {
            final List<ManifestEntry> manifestEntries = manifest.getManifestEntries();
            Session<UploadUnit> uploadSession = null;
            try {
                uploadSession = uploadTracker.submit(backuper, operation, manifestEntries, request.snapshotTag, operation.request.concurrentConnections);
                uploadSession.waitUntilConsideredFinished();
                uploadTracker.cancelIfNecessary(uploadSession);
                final List<UploadUnit> failedUnits = uploadSession.getFailedUnits();
                if (!failedUnits.isEmpty()) {
                    final String message = failedUnits.stream().map(unit -> unit.getManifestEntry().objectKey.toString()).collect(Collectors.joining(","));
                    logger.error(message);
                    throw new IOException(format("Unable to upload some files successfully: %s", message));
                }
            } finally {
                uploadTracker.removeSession(uploadSession);
                uploadSession = null;
            }
            if (operation.request.uploadClusterTopology) {
                // here we will upload all topology because we do not know what restore might look like (what dc a restorer will restore against if any)
                final ClusterTopology topology = new CassandraClusterTopology(cassandraJMXService, null).act();
                ClusterTopology.upload(backuper, topology, objectMapper, operation.request.snapshotTag);
            }
        } finally {
            manifest.cleanup();
        }
    } catch (final Exception ex) {
        operation.addError(Error.from(ex));
    } finally {
        final ClearSnapshotOperation cso = new ClearSnapshotOperation(cassandraJMXService, new ClearSnapshotOperationRequest(request.snapshotTag));
        try {
            cso.run0();
        } catch (final Exception ex) {
            operation.addErrors(cso.errors);
        }
    }
}

Example 2 with CassandraTokens

use of com.instaclustr.esop.impl.interaction.CassandraTokens in project esop by instaclustr.

the class ManifestTest method getManifest.

private Manifest getManifest(String snapshotName) throws Exception {
    waitForOperation(new TakeSnapshotOperation(jmx, new TakeSnapshotOperationRequest(DatabaseEntities.empty(), snapshotName), cassandraVersionProvider));
    Snapshots snapshot = Snapshots.parse(cassandraDataDir);
    Manifest manifest = new Manifest(snapshot.get(snapshotName).get());
    manifest.setSchemaVersion(new CassandraSchemaVersion(jmx).act());
    manifest.setTokens(new CassandraTokens(jmx).act());
    Thread.sleep(5000);
    return manifest;
}

Example 3 with CassandraTokens

use of com.instaclustr.esop.impl.interaction.CassandraTokens in project esop by instaclustr.

the class ManifestTest method testJsonManifest.

@Test
public void testJsonManifest() throws Exception {
    try {
        final List<String> tokens = new CassandraTokens(jmx).act();
        DatabaseEntities databaseEntities = DatabaseEntities.empty();
        // first table
        createTable("ks1", "ks1t1");
        disableAutocompaction("ks1");
        // #1 insert and flush & take snapshot
        insertDataIntoTable("ks1", "ks1t1");
        flush("ks1");
        insertDataIntoTable("ks1", "ks1t1");
        flush("ks1");
        waitForOperation(new TakeSnapshotOperation(jmx, new TakeSnapshotOperationRequest(databaseEntities, "snapshot1"), cassandraVersionProvider));
        // #2 insert and flush & take snapshot
        insertDataIntoTable("ks1", "ks1t1");
        flush("ks1");
        insertDataIntoTable("ks1", "ks1t1");
        flush("ks1");
        waitForOperation(new TakeSnapshotOperation(jmx, new TakeSnapshotOperationRequest(databaseEntities, "snapshot2"), cassandraVersionProvider));
        // second table
        createTable("ks2", "ks2t1");
        disableAutocompaction("ks2");
        // #1 insert and flush & take snapshot
        insertDataIntoTable("ks2", "ks2t1");
        flush("ks2");
        insertDataIntoTable("ks2", "ks2t1");
        flush("ks2");
        waitForOperation(new TakeSnapshotOperation(jmx, new TakeSnapshotOperationRequest(databaseEntities, "snapshot3"), cassandraVersionProvider));
        // parse
        final Snapshots snapshots = Snapshots.parse(cassandraDataDir);
        assertNotNull(snapshots);
        assertFalse(snapshots.isEmpty());
        assertEquals(3, snapshots.size());
        assertTrue(snapshots.get("snapshot1").isPresent());
        assertTrue(snapshots.get("snapshot2").isPresent());
        assertTrue(snapshots.get("snapshot3").isPresent());
        Manifest manifest = new Manifest(snapshots.get("snapshot3").get());
        // manifest itself, but it wont be serialised
        final Path localManifestPath = getLocalManifestPath("snapshot1");
        manifest.setManifest(getManifestAsManifestEntry(localManifestPath));
        // tokens
        manifest.setTokens(tokens);
        final String schemaVersion = new CassandraSchemaVersion(jmx).act();
        manifest.setSchemaVersion(schemaVersion);
        String writtenManifestAsJson = Manifest.write(manifest, objectMapper);
        logger.info(writtenManifestAsJson);
        assertNotNull(writtenManifestAsJson);
        Snapshot snapshot3 = snapshots.get("snapshot3").get();
        Optional<Keyspace> ks2 = snapshot3.getKeyspace("ks2");
        assertTrue(ks2.isPresent());
        assertTrue(ks2.get().containsTable("ks2t1"));
        List<ManifestEntry> ks2t1 = ks2.get().getManifestEntries("ks2t1");
        assertFalse(ks2t1.isEmpty());
        Manifest readManifest = Manifest.read(writtenManifestAsJson, objectMapper);
        assertNotNull(readManifest);
        HashMultimap<String, String> ksAndTables = readManifest.getSnapshot().getKeyspacesAndTables();
        // also system
        assertTrue(ksAndTables.size() > 2);
        assertTrue(ksAndTables.containsEntry("ks1", "ks1t1"));
        assertTrue(ksAndTables.containsEntry("ks2", "ks2t1"));
        HashMultimap<String, String> ksAndTablesWithoutSystem = readManifest.getSnapshot().getKeyspacesAndTables(false);
        assertEquals(2, ksAndTablesWithoutSystem.size());
        assertTrue(ksAndTables.containsEntry("ks1", "ks1t1"));
        assertTrue(ksAndTables.containsEntry("ks2", "ks2t1"));
        snapshots.clear();
        assertTrue(snapshots.isEmpty());
        assertEquals(0, snapshots.size());
    } finally {
        try {
            waitForOperation(new ClearSnapshotOperation(jmx, new ClearSnapshotOperationRequest("snapshot1")));
            waitForOperation(new ClearSnapshotOperation(jmx, new ClearSnapshotOperationRequest("snapshot2")));
            waitForOperation(new ClearSnapshotOperation(jmx, new ClearSnapshotOperationRequest("snapshot3")));
        } catch (final Exception ex) {
            logger.error("Unable to clear snapshots", ex);
        }
    }
}