Search in sources :

Example 1 with CassandraData

use of com.instaclustr.esop.impl.CassandraData in project esop by instaclustr.

the class BaseBackupOperationCoordinator method coordinate.

@Override
public void coordinate(final Operation<BackupOperationRequest> operation) {
    final BackupOperationRequest request = operation.request;
    logger.info(request.toString());
    try {
        assert cassandraJMXService != null;
        assert backuperFactoryMap != null;
        assert bucketServiceFactoryMap != null;
        assert objectMapper != null;
        if (!request.skipBucketVerification) {
            try (final BucketService bucketService = bucketServiceFactoryMap.get(request.storageLocation.storageProvider).createBucketService(request)) {
                bucketService.checkBucket(request.storageLocation.bucket, request.createMissingBucket);
            }
        }
        final CassandraData cassandraData = CassandraData.parse(request.dataDirs.get(0));
        cassandraData.setDatabaseEntitiesFromRequest(request.entities);
        final List<String> tokens = new CassandraTokens(cassandraJMXService).act();
        logger.info("Tokens {}", tokens);
        if (!Snapshots.snapshotContainsTimestamp(operation.request.snapshotTag)) {
            if (operation.request.schemaVersion == null) {
                operation.request.schemaVersion = new CassandraSchemaVersion(cassandraJMXService).act();
            }
            operation.request.snapshotTag = resolveSnapshotTag(operation.request, System.currentTimeMillis());
        }
        logger.info("Taking snapshot with name {}", request.snapshotTag);
        new TakeSnapshotOperation(cassandraJMXService, new TakeSnapshotOperationRequest(request.entities, request.snapshotTag), cassandraVersionProvider).run0();
        Snapshots.hashSpec = hashSpec;
        final Snapshots snapshots = Snapshots.parse(request.dataDirs, request.snapshotTag);
        final Optional<Snapshot> snapshot = snapshots.get(request.snapshotTag);
        if (!snapshot.isPresent()) {
            throw new IllegalStateException(format("There is not any snapshot of tag %s", request.snapshotTag));
        }
        final Manifest manifest = Manifest.from(snapshot.get());
        manifest.setSchemaVersion(request.schemaVersion);
        manifest.setTokens(tokens);
        // manifest
        final Path localManifestPath = getLocalManifestPath(request.snapshotTag);
        Manifest.write(manifest, localManifestPath, objectMapper);
        manifest.setManifest(getManifestAsManifestEntry(localManifestPath));
        try (final Backuper backuper = backuperFactoryMap.get(request.storageLocation.storageProvider).createBackuper(request)) {
            final List<ManifestEntry> manifestEntries = manifest.getManifestEntries();
            Session<UploadUnit> uploadSession = null;
            try {
                uploadSession = uploadTracker.submit(backuper, operation, manifestEntries, request.snapshotTag, operation.request.concurrentConnections);
                uploadSession.waitUntilConsideredFinished();
                uploadTracker.cancelIfNecessary(uploadSession);
                final List<UploadUnit> failedUnits = uploadSession.getFailedUnits();
                if (!failedUnits.isEmpty()) {
                    final String message = failedUnits.stream().map(unit -> unit.getManifestEntry().objectKey.toString()).collect(Collectors.joining(","));
                    logger.error(message);
                    throw new IOException(format("Unable to upload some files successfully: %s", message));
                }
            } finally {
                uploadTracker.removeSession(uploadSession);
                uploadSession = null;
            }
            if (operation.request.uploadClusterTopology) {
                // here we will upload all topology because we do not know what restore might look like (what dc a restorer will restore against if any)
                final ClusterTopology topology = new CassandraClusterTopology(cassandraJMXService, null).act();
                ClusterTopology.upload(backuper, topology, objectMapper, operation.request.snapshotTag);
            }
        } finally {
            manifest.cleanup();
        }
    } catch (final Exception ex) {
        operation.addError(Error.from(ex));
    } finally {
        final ClearSnapshotOperation cso = new ClearSnapshotOperation(cassandraJMXService, new ClearSnapshotOperationRequest(request.snapshotTag));
        try {
            cso.run0();
        } catch (final Exception ex) {
            operation.addErrors(cso.errors);
        }
    }
}
Also used : CassandraData(com.instaclustr.esop.impl.CassandraData) ManifestEntry(com.instaclustr.esop.impl.ManifestEntry) Error(com.instaclustr.operations.Operation.Error) Manifest.getManifestAsManifestEntry(com.instaclustr.esop.impl.Manifest.getManifestAsManifestEntry) Provider(javax.inject.Provider) HashSpec(com.instaclustr.esop.impl.hash.HashSpec) OperationCoordinator(com.instaclustr.operations.OperationCoordinator) LoggerFactory(org.slf4j.LoggerFactory) Manifest.getLocalManifestPath(com.instaclustr.esop.impl.Manifest.getLocalManifestPath) BackuperFactory(com.instaclustr.esop.guice.BackuperFactory) Map(java.util.Map) Session(com.instaclustr.esop.impl.AbstractTracker.Session) CassandraClusterTopology(com.instaclustr.esop.topology.CassandraClusterTopology) Path(java.nio.file.Path) CassandraData(com.instaclustr.esop.impl.CassandraData) BucketServiceFactory(com.instaclustr.esop.guice.BucketServiceFactory) BucketService(com.instaclustr.esop.impl.BucketService) Backuper(com.instaclustr.esop.impl.backup.Backuper) CassandraSchemaVersion(com.instaclustr.esop.impl.interaction.CassandraSchemaVersion) Logger(org.slf4j.Logger) UploadTracker(com.instaclustr.esop.impl.backup.UploadTracker) Operation(com.instaclustr.operations.Operation) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) IOException(java.io.IOException) Manifest(com.instaclustr.esop.impl.Manifest) Snapshot(com.instaclustr.esop.impl.Snapshots.Snapshot) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) ClearSnapshotOperationRequest(com.instaclustr.esop.impl.backup.coordination.ClearSnapshotOperation.ClearSnapshotOperationRequest) List(java.util.List) Snapshots(com.instaclustr.esop.impl.Snapshots) UploadUnit(com.instaclustr.esop.impl.backup.UploadTracker.UploadUnit) TakeSnapshotOperationRequest(com.instaclustr.esop.impl.backup.coordination.TakeSnapshotOperation.TakeSnapshotOperationRequest) CassandraVersion(com.instaclustr.cassandra.CassandraVersion) Optional(java.util.Optional) BackupOperationRequest(com.instaclustr.esop.impl.backup.BackupOperationRequest) CassandraTokens(com.instaclustr.esop.impl.interaction.CassandraTokens) ClusterTopology(com.instaclustr.esop.topology.CassandraClusterTopology.ClusterTopology) CassandraJMXService(jmx.org.apache.cassandra.service.CassandraJMXService) ClearSnapshotOperationRequest(com.instaclustr.esop.impl.backup.coordination.ClearSnapshotOperation.ClearSnapshotOperationRequest) TakeSnapshotOperationRequest(com.instaclustr.esop.impl.backup.coordination.TakeSnapshotOperation.TakeSnapshotOperationRequest) CassandraClusterTopology(com.instaclustr.esop.topology.CassandraClusterTopology) ClusterTopology(com.instaclustr.esop.topology.CassandraClusterTopology.ClusterTopology) CassandraSchemaVersion(com.instaclustr.esop.impl.interaction.CassandraSchemaVersion) UploadUnit(com.instaclustr.esop.impl.backup.UploadTracker.UploadUnit) CassandraClusterTopology(com.instaclustr.esop.topology.CassandraClusterTopology) CassandraTokens(com.instaclustr.esop.impl.interaction.CassandraTokens) Backuper(com.instaclustr.esop.impl.backup.Backuper) Snapshots(com.instaclustr.esop.impl.Snapshots) Manifest.getLocalManifestPath(com.instaclustr.esop.impl.Manifest.getLocalManifestPath) Path(java.nio.file.Path) BucketService(com.instaclustr.esop.impl.BucketService) IOException(java.io.IOException) Manifest(com.instaclustr.esop.impl.Manifest) IOException(java.io.IOException) Snapshot(com.instaclustr.esop.impl.Snapshots.Snapshot) BackupOperationRequest(com.instaclustr.esop.impl.backup.BackupOperationRequest) ManifestEntry(com.instaclustr.esop.impl.ManifestEntry) Manifest.getManifestAsManifestEntry(com.instaclustr.esop.impl.Manifest.getManifestAsManifestEntry)

Example 2 with CassandraData

use of com.instaclustr.esop.impl.CassandraData in project esop by instaclustr.

the class CassandraDataTest method testCassandraDataMissingFromInRenamingInEntitiesShouldFail.

@Test
public // --entities=ks1.tb1 --rename=ks1.tb2=ks1.tb2 -> invalid as "from" is not in entities
void testCassandraDataMissingFromInRenamingInEntitiesShouldFail() {
    CassandraData parsed = new CassandraData(tableIdsMap, paths);
    parsed.setDatabaseEntitiesFromRequest(DatabaseEntities.parse("ks1  .  tb1"));
    parsed.setRenamedEntitiesFromRequest(RenamedEntities.parse(new HashMap<String, String>() {

        {
            put("ks1.tb2", "ks1.tb3");
        }
    }));
    try {
        parsed.validate();
        fail("setting rename entities where 'from' is missing in entities should fail");
    } catch (final Exception ex) {
    // ok
    }
}
Also used : CassandraData(com.instaclustr.esop.impl.CassandraData) HashMap(java.util.HashMap) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest)

Example 3 with CassandraData

use of com.instaclustr.esop.impl.CassandraData in project esop by instaclustr.

the class CassandraDataTest method testCassandraData.

@Test
public void testCassandraData() {
    CassandraData parsed = new CassandraData(tableIdsMap, paths);
    assertTrue(parsed.getTableId("system_schema", "tables").isPresent());
    assertFalse(parsed.getTableId("system_schema", "asdadsad").isPresent());
    assertFalse(parsed.getTableId("sdadasd", "types").isPresent());
    Optional<List<String>> systemSchemaTablesNames = parsed.getTablesNames("system_schema");
    assertTrue(systemSchemaTablesNames.isPresent());
    Optional<List<Path>> systemSchemaTablesPaths = parsed.getTablesPaths("system_schema");
    assertTrue(systemSchemaTablesPaths.isPresent());
    assertEquals(2, systemSchemaTablesPaths.get().size());
    assertTrue(parsed.containsKeyspace("ks1"));
    assertFalse(parsed.containsKeyspace("asdsad"));
    assertTrue(parsed.containsTable("ks1", "tb1"));
    assertTrue(parsed.containsTable("ks1", "tb2"));
    assertFalse(parsed.containsTable("ks1", "sdada"));
    assertEquals("11111111111111111111111111111111", parsed.getTableId("ks1", "tb1").get());
    assertEquals("88888888888888888888888888888888", parsed.getTableId("ks2", "tb4").get());
    assertFalse(parsed.getTableId("ks2", "sdadsad").isPresent());
    assertTrue(parsed.getTablesNames("ks1").get().containsAll(Arrays.asList("tb1", "tb2", "tb3", "tb4")));
    assertTrue(parsed.getTablesNames("ks2").get().containsAll(Arrays.asList("tb1", "tb2", "tb3", "tb4")));
    assertTrue(parsed.getTablesPaths("ks1").get().containsAll(Arrays.asList(Paths.get("/var/lib/cassandra/data/data/ks1/tb1-11111111111111111111111111111111"), Paths.get("/var/lib/cassandra/data/data/ks1/tb2-22222222222222222222222222222222"), Paths.get("/var/lib/cassandra/data/data/ks1/tb3-33333333333333333333333333333333"), Paths.get("/var/lib/cassandra/data/data/ks1/tb4-44444444444444444444444444444444"))));
    assertTrue(parsed.getTablePath("ks1", "tb3").get().equals(Paths.get("/var/lib/cassandra/data/data/ks1/tb3-33333333333333333333333333333333")));
    assertFalse(parsed.getTablePath("ks1", "sdad").isPresent());
    assertFalse(parsed.getTablePath("sdad", "sdad").isPresent());
    assertEquals(Paths.get("/var/lib/cassandra/data/data/ks1"), parsed.getKeyspacePath("ks1").get());
    assertEquals(3, parsed.getKeyspacePaths().size());
    // database entities
    DatabaseEntities databaseEntities = parsed.toDatabaseEntities(false);
    assertFalse(databaseEntities.areEmpty());
    assertTrue(databaseEntities.getKeyspaces().containsAll(Arrays.asList("ks1", "ks2")));
    databaseEntities.getKeyspacesAndTables().entries().forEach(entry -> {
        assertTrue(parsed.getTableId(entry.getKey(), entry.getValue()).isPresent());
    });
    assertFalse(databaseEntities.contains("system_schema"));
    assertFalse(databaseEntities.contains("asdasda", "asdad"));
}
Also used : CassandraData(com.instaclustr.esop.impl.CassandraData) DatabaseEntities(com.instaclustr.esop.impl.DatabaseEntities) List(java.util.List) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest)

Example 4 with CassandraData

use of com.instaclustr.esop.impl.CassandraData in project esop by instaclustr.

the class CassandraDataTest method testDatabaseEntities.

@Test
public void testDatabaseEntities() throws Exception {
    CassandraData parsed = new CassandraData(tableIdsMap, paths);
    DatabaseEntities entities = DatabaseEntities.parse("ks1  .tb1  ,  kb3.  tb3");
    try {
        parsed.setDatabaseEntitiesFromRequest(entities);
    } catch (Exception ex) {
        assertEquals(ex.getMessage(), "Tables [kb3.tb3] to process are not present in Cassandra.");
    }
    parsed.setDatabaseEntitiesFromRequest(DatabaseEntities.parse("ks1  .tb1"));
}
Also used : CassandraData(com.instaclustr.esop.impl.CassandraData) DatabaseEntities(com.instaclustr.esop.impl.DatabaseEntities) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest)

Example 5 with CassandraData

use of com.instaclustr.esop.impl.CassandraData in project esop by instaclustr.

the class CassandraDataTest method testCassandraDataForRenamingEntities.

@Test
public // --entities=ks1.tb1,ks1.tb2 --rename=ks1.tb2=ks1.tb1 -> invalid as "to" is in entities
void testCassandraDataForRenamingEntities() {
    CassandraData parsed = new CassandraData(tableIdsMap, paths);
    parsed.setDatabaseEntitiesFromRequest(DatabaseEntities.parse("ks1.tb1,  ks1.tb3,   ks2.tb3,   ks2.tb4"));
    parsed.setRenamedEntitiesFromRequest(RenamedEntities.parse(new HashMap<String, String>() {

        {
            put("ks1.tb1", "ks1.tb2");
            put("ks2.tb3", "ks2.tb2");
        }
    }));
    DatabaseEntities databaseEntitiesToProcessForRestore = parsed.getDatabaseEntitiesToProcessForRestore();
    assertTrue(databaseEntitiesToProcessForRestore.contains("ks1", "tb2"));
    assertTrue(databaseEntitiesToProcessForRestore.contains("ks2", "tb2"));
    assertTrue(databaseEntitiesToProcessForRestore.contains("ks1", "tb3"));
    assertTrue(databaseEntitiesToProcessForRestore.contains("ks2", "tb4"));
    assertFalse(databaseEntitiesToProcessForRestore.contains("ks1", "tb1"));
    assertFalse(databaseEntitiesToProcessForRestore.contains("ks2", "tb3"));
    assertEquals(2, databaseEntitiesToProcessForRestore.getKeyspaces().size());
    assertEquals(4, databaseEntitiesToProcessForRestore.getKeyspacesAndTables().size());
}
Also used : CassandraData(com.instaclustr.esop.impl.CassandraData) DatabaseEntities(com.instaclustr.esop.impl.DatabaseEntities) HashMap(java.util.HashMap) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest)

Aggregations

CassandraData (com.instaclustr.esop.impl.CassandraData)10 BeforeTest (org.testng.annotations.BeforeTest)9 Test (org.testng.annotations.Test)9 HashMap (java.util.HashMap)5 DatabaseEntities (com.instaclustr.esop.impl.DatabaseEntities)4 List (java.util.List)2 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 CassandraVersion (com.instaclustr.cassandra.CassandraVersion)1 BackuperFactory (com.instaclustr.esop.guice.BackuperFactory)1 BucketServiceFactory (com.instaclustr.esop.guice.BucketServiceFactory)1 Session (com.instaclustr.esop.impl.AbstractTracker.Session)1 BucketService (com.instaclustr.esop.impl.BucketService)1 Manifest (com.instaclustr.esop.impl.Manifest)1 Manifest.getLocalManifestPath (com.instaclustr.esop.impl.Manifest.getLocalManifestPath)1 Manifest.getManifestAsManifestEntry (com.instaclustr.esop.impl.Manifest.getManifestAsManifestEntry)1 ManifestEntry (com.instaclustr.esop.impl.ManifestEntry)1 Snapshots (com.instaclustr.esop.impl.Snapshots)1 Snapshot (com.instaclustr.esop.impl.Snapshots.Snapshot)1 BackupOperationRequest (com.instaclustr.esop.impl.backup.BackupOperationRequest)1 Backuper (com.instaclustr.esop.impl.backup.Backuper)1