Examples with ManifestEntry com.instaclustr.esop.impl.ManifestEntry used on opensource projects

Example 11 with ManifestEntry

use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.

the class AllManifestsReportTest method testSerialization.

@Test
public void testSerialization() {
    Manifest.AllManifestsReport report = new Manifest.AllManifestsReport();
    report.totalFiles = 10;
    report.totalSize = 500;
    report.totalManifests = 10;
    Manifest.ManifestReporter.ManifestReport manifestReport = new Manifest.ManifestReporter.ManifestReport();
    manifestReport.name = "123";
    manifestReport.reclaimableSpace = 100;
    manifestReport.size = 10;
    manifestReport.files = 3;
    manifestReport.unixtimestamp = 124L;
    manifestReport.removableEntries = Collections.singletonList("123");
    manifestReport.manifest = new ManifestEntry(Paths.get("somekey"), Paths.get("localFile"), ManifestEntry.Type.MANIFEST_FILE, 100, new KeyspaceTable("ks1", "tb1"), "123");
    report.reports = Collections.singletonList(manifestReport);
    ObjectMapper mapper = new ObjectMapper();
    try {
        String s = mapper.writeValueAsString(report);
        Manifest.AllManifestsReport report1 = mapper.readValue(s, Manifest.AllManifestsReport.class);
        System.out.println(report1);
    } catch (final Exception ex) {
        throw new RuntimeException(ex);
    }
}

Example 12 with ManifestEntry

use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.

the class ManifestComponentsTest method testManifestFilterNoRestoreSystemKeyspaceNoNewClusterNoEmptyRequestEntities.

@Test
public void testManifestFilterNoRestoreSystemKeyspaceNoNewClusterNoEmptyRequestEntities() throws Exception {
    Manifest manifest = parseManifest();
    manifest.enrichManifestEntries();
    List<ManifestEntry> manifestFiles = manifest.getManifestFiles(DatabaseEntities.parse("ks1"), // restoreSystemKeyspace
    false, // restoreSystemAuth
    false, // newCluster
    false, // withSchemas
    false);
    Assert.assertTrue(manifestFiles.stream().noneMatch(entry -> KeyspaceTable.isSystemKeyspace(entry.keyspaceTable.keyspace)));
    Assert.assertTrue(manifestFiles.stream().allMatch(entry -> entry.keyspaceTable.keyspace.equals("ks1")));
}

Example 13 with ManifestEntry

use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.

the class ManifestComponentsTest method testManifestFilterNoRestoreSystemKeyspaceNoNewCluster.

/**
 * ks1
 *  ks1t1
 *  ks1t2
 * ks2
 *  ks2t1
 *  ks2t2
 * system_auth
 * system
 *
 * @throws Exception
 */
@Test
public void testManifestFilterNoRestoreSystemKeyspaceNoNewCluster() throws Exception {
    Manifest manifest = parseManifest();
    manifest.enrichManifestEntries();
    List<ManifestEntry> manifestFiles = manifest.getManifestFiles(new DatabaseEntities(), // restoreSystemKeyspace
    false, // restoreSystemAuth
    false, // newCluster
    false, // withSchemas
    false);
    Assert.assertTrue(manifestFiles.stream().noneMatch(entry -> KeyspaceTable.isSystemKeyspace(entry.keyspaceTable.keyspace)));
}

Example 14 with ManifestEntry

use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.

the class ManifestTest method testJsonManifest.

@Test
public void testJsonManifest() throws Exception {
    try {
        final List<String> tokens = new CassandraTokens(jmx).act();
        DatabaseEntities databaseEntities = DatabaseEntities.empty();
        // first table
        createTable("ks1", "ks1t1");
        disableAutocompaction("ks1");
        // #1 insert and flush & take snapshot
        insertDataIntoTable("ks1", "ks1t1");
        flush("ks1");
        insertDataIntoTable("ks1", "ks1t1");
        flush("ks1");
        waitForOperation(new TakeSnapshotOperation(jmx, new TakeSnapshotOperationRequest(databaseEntities, "snapshot1"), cassandraVersionProvider));
        // #2 insert and flush & take snapshot
        insertDataIntoTable("ks1", "ks1t1");
        flush("ks1");
        insertDataIntoTable("ks1", "ks1t1");
        flush("ks1");
        waitForOperation(new TakeSnapshotOperation(jmx, new TakeSnapshotOperationRequest(databaseEntities, "snapshot2"), cassandraVersionProvider));
        // second table
        createTable("ks2", "ks2t1");
        disableAutocompaction("ks2");
        // #1 insert and flush & take snapshot
        insertDataIntoTable("ks2", "ks2t1");
        flush("ks2");
        insertDataIntoTable("ks2", "ks2t1");
        flush("ks2");
        waitForOperation(new TakeSnapshotOperation(jmx, new TakeSnapshotOperationRequest(databaseEntities, "snapshot3"), cassandraVersionProvider));
        // parse
        final Snapshots snapshots = Snapshots.parse(cassandraDataDir);
        assertNotNull(snapshots);
        assertFalse(snapshots.isEmpty());
        assertEquals(3, snapshots.size());
        assertTrue(snapshots.get("snapshot1").isPresent());
        assertTrue(snapshots.get("snapshot2").isPresent());
        assertTrue(snapshots.get("snapshot3").isPresent());
        Manifest manifest = new Manifest(snapshots.get("snapshot3").get());
        // manifest itself, but it wont be serialised
        final Path localManifestPath = getLocalManifestPath("snapshot1");
        manifest.setManifest(getManifestAsManifestEntry(localManifestPath));
        // tokens
        manifest.setTokens(tokens);
        final String schemaVersion = new CassandraSchemaVersion(jmx).act();
        manifest.setSchemaVersion(schemaVersion);
        String writtenManifestAsJson = Manifest.write(manifest, objectMapper);
        logger.info(writtenManifestAsJson);
        assertNotNull(writtenManifestAsJson);
        Snapshot snapshot3 = snapshots.get("snapshot3").get();
        Optional<Keyspace> ks2 = snapshot3.getKeyspace("ks2");
        assertTrue(ks2.isPresent());
        assertTrue(ks2.get().containsTable("ks2t1"));
        List<ManifestEntry> ks2t1 = ks2.get().getManifestEntries("ks2t1");
        assertFalse(ks2t1.isEmpty());
        Manifest readManifest = Manifest.read(writtenManifestAsJson, objectMapper);
        assertNotNull(readManifest);
        HashMultimap<String, String> ksAndTables = readManifest.getSnapshot().getKeyspacesAndTables();
        // also system
        assertTrue(ksAndTables.size() > 2);
        assertTrue(ksAndTables.containsEntry("ks1", "ks1t1"));
        assertTrue(ksAndTables.containsEntry("ks2", "ks2t1"));
        HashMultimap<String, String> ksAndTablesWithoutSystem = readManifest.getSnapshot().getKeyspacesAndTables(false);
        assertEquals(2, ksAndTablesWithoutSystem.size());
        assertTrue(ksAndTables.containsEntry("ks1", "ks1t1"));
        assertTrue(ksAndTables.containsEntry("ks2", "ks2t1"));
        snapshots.clear();
        assertTrue(snapshots.isEmpty());
        assertEquals(0, snapshots.size());
    } finally {
        try {
            waitForOperation(new ClearSnapshotOperation(jmx, new ClearSnapshotOperationRequest("snapshot1")));
            waitForOperation(new ClearSnapshotOperation(jmx, new ClearSnapshotOperationRequest("snapshot2")));
            waitForOperation(new ClearSnapshotOperation(jmx, new ClearSnapshotOperationRequest("snapshot3")));
        } catch (final Exception ex) {
            logger.error("Unable to clear snapshots", ex);
        }
    }
}

Example 15 with ManifestEntry

use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.

the class DataSynchronizator method execute.

public DataSynchronizator execute() {
    final DatabaseEntities filteredManifestDatabaseEntities = manifest.getDatabaseEntities(true).filter(request.entities);
    logger.info("filtered entities from manifest: " + filteredManifestDatabaseEntities.toString());
    final List<ManifestEntry> entriesFromManifest = manifest.getManifestFiles(filteredManifestDatabaseEntities, request.restoreSystemKeyspace, request.restoreSystemAuth, request.newCluster, true);
    // 3. Build a list of all SSTables files currently present (that are candidates for deleting)
    final Set<Path> localDataFiles = CassandraData.getLocalDataFiles(request.dirs.data());
    logger.info("Restoring to existing cluster: {}", localDataFiles.size() > 0);
    for (final ManifestEntry entryFromManifest : entriesFromManifest) {
        // do not download schemas
        if (entryFromManifest.type == ManifestEntry.Type.CQL_SCHEMA) {
            continue;
        }
        if (CassandraData.containsFile(localDataFiles, entryFromManifest.localFile)) {
            // this file exists on a local disk as well as in manifest, there is nothing to download nor remove
            logger.info(String.format("%s found locally, not downloading", entryFromManifest.localFile));
        } else {
            // if it does not exist locally, we have to download it
            entriesToDownload.add(entryFromManifest);
        }
    }
    for (final Path localExistingFile : localDataFiles) {
        final Optional<ManifestEntry> first = entriesFromManifest.stream().filter(me -> localExistingFile.endsWith(me.localFile)).findFirst();
        if (first.isPresent()) {
            // if it exists, hash has to be same, otherwise delete it
            if (!SSTableUtils.isExistingSStable(localExistingFile, first.get().objectKey.getName(SSTableUtils.isSecondaryIndexManifest(first.get().objectKey) ? 4 : 3).toString())) {
                filesToDelete.add(localExistingFile);
            }
        } else {
            filesToDelete.add(localExistingFile);
        }
    }
    return this;
}