Examples with ManifestEntry com.instaclustr.esop.impl.ManifestEntry used on opensource projects

Example 6 with ManifestEntry

use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.

the class ManifestComponentsTest method testManifestFilterNoRestoreSystemKeyspaceNoNewClusterTableRequestEntities.

@Test
public void testManifestFilterNoRestoreSystemKeyspaceNoNewClusterTableRequestEntities() throws Exception {
    Manifest manifest = parseManifest();
    manifest.enrichManifestEntries();
    List<ManifestEntry> manifestFiles = manifest.getManifestFiles(DatabaseEntities.parse("ks1.ks1t2"), // restoreSystemKeyspace
    false, // restoreSystemAuth
    false, // newCluster
    false, // withSchemas
    false);
    Assert.assertTrue(manifestFiles.stream().noneMatch(entry -> KeyspaceTable.isSystemKeyspace(entry.keyspaceTable.keyspace)));
    Assert.assertTrue(manifestFiles.stream().allMatch(entry -> entry.keyspaceTable.keyspace.equals("ks1") || entry.keyspaceTable.table.equals("ks1t2")));
}

Example 7 with ManifestEntry

use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.

the class ManifestComponentsTest method testTableEquality.

@Test
public void testTableEquality() throws Exception {
    Manifest manifest = parseManifest();
    Snapshot snapshot = manifest.getSnapshot();
    Keyspace keyspace = snapshot.getKeyspace("ks1").orElseThrow(IllegalStateException::new);
    Table table = keyspace.getTable("ks1t1").orElseThrow(IllegalStateException::new);
    Table cloned = table.clone();
    assertNotSame(table, cloned);
    assertEquals(table, cloned);
    cloned.add("me-1-big-Data.db", new ManifestEntry(Paths.get("abc/def"), null, Type.FILE, 50, null, null));
    assertNotEquals(table, cloned);
}

Example 8 with ManifestEntry

use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.

the class ManifestComponentsTest method testManifestEntry.

@Test
public void testManifestEntry() throws CloneNotSupportedException {
    ManifestEntry original = new ManifestEntry(Paths.get("object/key"), Paths.get("local/file"), Type.FILE, null, new KeyspaceTable("keyspace", "table"));
    ManifestEntry cloned = original.clone();
    assertNotEquals(original, cloned);
    original.localFile = Paths.get("other/file");
    original.objectKey = Paths.get("other/object");
    original.type = Type.CQL_SCHEMA;
    original.keyspaceTable = new KeyspaceTable("k2", "t2");
    assertNotEquals(original, cloned);
}

Example 9 with ManifestEntry

use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.

the class LocalBackupTest method testUploadTracker.

@Test
public void testUploadTracker() throws Exception {
    final String snapshotName = UUID.randomUUID().toString();
    final String snapshotName2 = UUID.randomUUID().toString();
    List<Path> dataDirs = Arrays.asList(cassandraDir.toAbsolutePath().resolve("data").resolve("data"), cassandraDir.toAbsolutePath().resolve("data").resolve("data2"), cassandraDir.toAbsolutePath().resolve("data").resolve("data3"));
    final BackupOperationRequest backupOperationRequest = getBackupOperationRequestForTracker(snapshotName, "test,test2", dataDirs);
    final BackupOperationRequest backupOperationRequest2 = getBackupOperationRequestForTracker(snapshotName2, "test", dataDirs);
    UploadTracker uploadTracker = null;
    Cassandra cassandra = null;
    try {
        cassandra = getCassandra(cassandraDir, CASSANDRA_VERSION);
        cassandra.start();
        try (CqlSession session = CqlSession.builder().build()) {
            assertEquals(populateDatabase(session).size(), NUMBER_OF_INSERTED_ROWS);
        }
        final AtomicBoolean wait = new AtomicBoolean(true);
        final ListeningExecutorService finisher = new Executors.FixedTasksExecutorSupplier().get(10);
        uploadTracker = new UploadTracker(finisher, operationsService, new HashSpec()) {

            // override for testing purposes
            @Override
            public UploadUnit constructUnitToSubmit(final Backuper backuper, final ManifestEntry manifestEntry, final AtomicBoolean shouldCancel, final String snapshotTag, final HashSpec hashSpec) {
                return new TestingUploadUnit(wait, backuper, manifestEntry, shouldCancel, snapshotTag, hashSpec);
            }
        };
        final LocalFileBackuper backuper = new LocalFileBackuper(backupOperationRequest);
        new TakeSnapshotOperation(jmxService, new TakeSnapshotOperationRequest(backupOperationRequest.entities, backupOperationRequest.snapshotTag), cassandraVersionProvider).run();
        new TakeSnapshotOperation(jmxService, new TakeSnapshotOperationRequest(backupOperationRequest2.entities, backupOperationRequest2.snapshotTag), cassandraVersionProvider).run();
        final Snapshots snapshots = Snapshots.parse(dataDirs);
        final Optional<Snapshot> snapshot = snapshots.get(backupOperationRequest.snapshotTag);
        final Optional<Snapshot> snapshot2 = snapshots.get(backupOperationRequest2.snapshotTag);
        assert snapshot.isPresent();
        assert snapshot2.isPresent();
        Set<String> providers = Stream.of("file").collect(Collectors.toSet());
        final BackupOperation backupOperation = new BackupOperation(operationCoordinator, providers, backupOperationRequest);
        final BackupOperation backupOperation2 = new BackupOperation(operationCoordinator, providers, backupOperationRequest2);
        final List<ManifestEntry> manifestEntries = Manifest.from(snapshot.get()).getManifestEntries();
        final List<ManifestEntry> manifestEntries2 = Manifest.from(snapshot2.get()).getManifestEntries();
        Session<UploadUnit> session = uploadTracker.submit(backuper, backupOperation, manifestEntries, backupOperation.request.snapshotTag, backupOperation.request.concurrentConnections);
        final int submittedUnits1 = uploadTracker.submittedUnits.intValue();
        Assert.assertEquals(manifestEntries.size(), submittedUnits1);
        final Session<UploadUnit> session2 = uploadTracker.submit(backuper, backupOperation2, manifestEntries2, backupOperation.request.snapshotTag, backupOperation.request.concurrentConnections);
        final int submittedUnits2 = uploadTracker.submittedUnits.intValue();
        // even we submitted the second session, it does not change the number of units because session2
        // wants to upload "test" but it is already going to be uploaded by session1
        // we have effectively submitted only what should be submitted, no duplicates
        // so it is as if "test" from session2 was not submitted at all
        Assert.assertEquals(submittedUnits1, submittedUnits2);
        Assert.assertEquals(manifestEntries.size(), uploadTracker.submittedUnits.intValue());
        // however we have submitted two sessions in total
        Assert.assertEquals(2, uploadTracker.submittedSessions.intValue());
        // lets upload it now
        wait.set(false);
        session.waitUntilConsideredFinished();
        session2.waitUntilConsideredFinished();
        Assert.assertTrue(session.isConsideredFinished());
        Assert.assertTrue(session.isSuccessful());
        Assert.assertTrue(session.getFailedUnits().isEmpty());
        Assert.assertEquals(uploadTracker.submittedUnits.intValue(), session.getUnits().size());
        Assert.assertTrue(session2.isConsideredFinished());
        Assert.assertTrue(session2.isSuccessful());
        Assert.assertTrue(session2.getFailedUnits().isEmpty());
        Assert.assertTrue(submittedUnits2 > session2.getUnits().size());
        for (final UploadUnit uploadUnit : session2.getUnits()) {
            Assert.assertTrue(session.getUnits().contains(uploadUnit));
        }
        Assert.assertTrue(uploadTracker.getUnits().isEmpty());
        uploadTracker.removeSession(session);
        uploadTracker.removeSession(session2);
        Assert.assertTrue(session.getUnits().isEmpty());
        Assert.assertTrue(session2.getUnits().isEmpty());
    } catch (final Exception ex) {
        ex.printStackTrace();
        throw ex;
    } finally {
        new ClearSnapshotOperation(jmxService, new ClearSnapshotOperationRequest(backupOperationRequest.snapshotTag)).run();
        if (cassandra != null) {
            cassandra.stop();
        }
        uploadTracker.stopAsync();
        uploadTracker.awaitTerminated(1, MINUTES);
        uploadTracker.stopAsync();
        uploadTracker.awaitTerminated(1, MINUTES);
        FileUtils.deleteDirectory(Paths.get(target(backupOperationRequest.storageLocation.bucket)));
    }
}

Example 10 with ManifestEntry

use of com.instaclustr.esop.impl.ManifestEntry in project esop by instaclustr.

the class BackupRestoreTest method testSSTableLister.

@Test(description = "Test that the manifest is correctly constructed, includes expected files and generates checksum if necessary")
@Ignore
public void testSSTableLister() throws Exception {
    // TODO not sure why this doesn't recreate things fully given its called before each test
    hardResetTestDirs();
    for (TestFileConfig testFileConfig : versionsToTest) {
        Path backupRoot = Paths.get("/backupRoot/keyspace1");
        final String keyspace = "keyspace1";
        final String table1 = "table1";
        final Path table1Path = tempDirs.get(testFileConfig.cassandraVersion.toString()).resolve("data/" + keyspace + "/" + table1);
        Map<String, List<ManifestEntry>> sstables = SSTableUtils.getSSTables(keyspace, table1, table1Path, backupRoot.resolve(table1Path.getFileName()), new HashSpec());
        final String table2 = "table2";
        final Path table2Path = tempDirs.get(testFileConfig.cassandraVersion.toString()).resolve("data/" + keyspace + "/" + table2);
        sstables.putAll(SSTableUtils.getSSTables(keyspace, table2, table2Path, backupRoot.resolve(table2Path.getFileName()), new HashSpec()));
        Map<Path, Path> manifestMap = new HashMap<>();
        for (ManifestEntry e : sstables.values().stream().flatMap(Collection::stream).collect(Collectors.toList())) {
            manifestMap.put(e.localFile, e.objectKey);
        }
        if (CassandraVersion.isTwoZero(testFileConfig.cassandraVersion)) {
            // table1 is un-compressed so should have written out a sha1 digest
            final Path localPath1 = table1Path.resolve(String.format("%s-1-big-Data.db", testFileConfig.getSstablePrefix(keyspace, table1)));
            assertEquals(manifestMap.get(localPath1), backupRoot.resolve(String.format("%s/1-%s/%s-1-big-Data.db", table1, sha1Hash, testFileConfig.getSstablePrefix(keyspace, table1))));
            final Path localPath2 = table1Path.resolve(String.format("%s-3-big-Index.db", testFileConfig.getSstablePrefix(keyspace, table1)));
            final String checksum2 = SSTableUtils.calculateChecksum(localPath2);
            assertEquals(manifestMap.get(localPath2), backupRoot.resolve(String.format("%s/3-%s/%s-3-big-Index.db", table1, checksum2, testFileConfig.getSstablePrefix(keyspace, table1))));
            final Path localPath3 = table2Path.resolve(String.format("%s-1-big-Data.db", testFileConfig.getSstablePrefix(keyspace, table2)));
            final String checksum3 = SSTableUtils.calculateChecksum(localPath3);
            assertEquals(manifestMap.get(localPath3), backupRoot.resolve(String.format("%s/1-%s/%s-1-big-Data.db", table2, checksum3, testFileConfig.getSstablePrefix(keyspace, table2))));
            assertNull(manifestMap.get(table2Path.resolve(String.format("%s-3-big-Index.db", testFileConfig.getSstablePrefix(keyspace, table2)))));
        } else {
            Path resolve = table1Path.resolve(String.format("%s-1-big-Data.db", testFileConfig.getSstablePrefix(keyspace, table1)));
            assertEquals(manifestMap.get(resolve), backupRoot.resolve(String.format("%s/1-1000000000/%s-1-big-Data.db", table1, testFileConfig.getSstablePrefix(keyspace, table1))));
            // Cassandra doesn't create CRC32 file for 2.0.x
            assertEquals(manifestMap.get(table1Path.resolve(String.format("%s-2-big-Digest.crc32", testFileConfig.getSstablePrefix(keyspace, table1)))), backupRoot.resolve(String.format("%s/2-1000000000/%s-2-big-Digest.crc32", table1, testFileConfig.getSstablePrefix(keyspace, table1))));
            assertEquals(manifestMap.get(table1Path.resolve(String.format("%s-3-big-Index.db", testFileConfig.getSstablePrefix(keyspace, table1)))), backupRoot.resolve(String.format("%s/3-1000000000/%s-3-big-Index.db", table1, testFileConfig.getSstablePrefix(keyspace, table1))));
            assertEquals(manifestMap.get(table2Path.resolve(String.format("%s-1-big-Data.db", testFileConfig.getSstablePrefix(keyspace, table2)))), backupRoot.resolve(String.format("%s/1-1000000000/%s-1-big-Data.db", table2, testFileConfig.getSstablePrefix(keyspace, table2))));
            assertEquals(manifestMap.get(table2Path.resolve(String.format("%s-2-big-Digest.crc32", testFileConfig.getSstablePrefix(keyspace, table2)))), backupRoot.resolve(String.format("%s/2-1000000000/%s-2-big-Digest.crc32", table2, testFileConfig.getSstablePrefix(keyspace, table2))));
            assertNull(manifestMap.get(table2Path.resolve(String.format("%s-3-big-Index.db", testFileConfig.getSstablePrefix(keyspace, table2)))));
        }
        assertNull(manifestMap.get(table1Path.resolve("manifest.json")));
        assertNull(manifestMap.get(table1Path.resolve("backups")));
        assertNull(manifestMap.get(table1Path.resolve("snapshots")));
    }
}