Examples with SnapshotManifest org.apache.cassandra.service.snapshot.SnapshotManifest used on opensource projects

Example 1 with SnapshotManifest

use of org.apache.cassandra.service.snapshot.SnapshotManifest in project cassandra by apache.

the class Directories method listSnapshots.

public Map<String, TableSnapshot> listSnapshots() {
    Map<String, Set<File>> snapshotDirsByTag = listSnapshotDirsByTag();
    Map<String, TableSnapshot> snapshots = Maps.newHashMapWithExpectedSize(snapshotDirsByTag.size());
    for (Map.Entry<String, Set<File>> entry : snapshotDirsByTag.entrySet()) {
        String tag = entry.getKey();
        Set<File> snapshotDirs = entry.getValue();
        SnapshotManifest manifest = maybeLoadManifest(metadata.keyspace, metadata.name, tag, snapshotDirs);
        snapshots.put(tag, buildSnapshot(tag, manifest, snapshotDirs));
    }
    return snapshots;
}

Example 2 with SnapshotManifest

use of org.apache.cassandra.service.snapshot.SnapshotManifest in project cassandra by apache.

the class ColumnFamilyStoreTest method testSnapshotWithoutFlushWithSecondaryIndexes.

@Test
public void testSnapshotWithoutFlushWithSecondaryIndexes() throws Exception {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_INDEX1);
    cfs.truncateBlocking();
    UpdateBuilder builder = UpdateBuilder.create(cfs.metadata.get(), "key").newRow().add("birthdate", 1L).add("notbirthdate", 2L);
    new Mutation(builder.build()).applyUnsafe();
    cfs.forceBlockingFlush();
    String snapshotName = "newSnapshot";
    cfs.snapshotWithoutFlush(snapshotName);
    File snapshotManifestFile = cfs.getDirectories().getSnapshotManifestFile(snapshotName);
    SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(snapshotManifestFile);
    // Keyspace1-Indexed1 and the corresponding index
    assertThat(manifest.getFiles()).hasSize(2);
    // Snapshot of the secondary index is stored in the subfolder with the same file name
    String baseTableFile = manifest.getFiles().get(0);
    String indexTableFile = manifest.getFiles().get(1);
    assertThat(baseTableFile).isNotEqualTo(indexTableFile);
    assertThat(Directories.isSecondaryIndexFolder(new File(indexTableFile).parent())).isTrue();
    assertThat(indexTableFile).endsWith(baseTableFile);
}

Example 3 with SnapshotManifest

use of org.apache.cassandra.service.snapshot.SnapshotManifest in project cassandra by apache.

the class SASIIndexTest method testSASIComponentsAddedToSnapshot.

@Test
public void testSASIComponentsAddedToSnapshot() throws Throwable {
    String snapshotName = "sasi_test";
    Map<String, Pair<String, Integer>> data = new HashMap<>();
    Random r = new Random();
    for (int i = 0; i < 100; i++) data.put(UUID.randomUUID().toString(), Pair.create(UUID.randomUUID().toString(), r.nextInt()));
    ColumnFamilyStore store = loadData(data, true);
    store.forceMajorCompaction();
    Set<SSTableReader> ssTableReaders = store.getLiveSSTables();
    Set<Component> sasiComponents = new HashSet<>();
    for (Index index : store.indexManager.listIndexes()) if (index instanceof SASIIndex)
        sasiComponents.add(((SASIIndex) index).getIndex().getComponent());
    Assert.assertFalse(sasiComponents.isEmpty());
    try {
        store.snapshot(snapshotName);
        SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(store.getDirectories().getSnapshotManifestFile(snapshotName));
        Assert.assertFalse(ssTableReaders.isEmpty());
        Assert.assertFalse(manifest.files.isEmpty());
        Assert.assertEquals(ssTableReaders.size(), manifest.files.size());
        Map<Descriptor, Set<Component>> snapshotSSTables = store.getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots(snapshotName).list();
        long indexSize = 0;
        long tableSize = 0;
        for (SSTableReader sstable : ssTableReaders) {
            File snapshotDirectory = Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
            Descriptor snapshotSSTable = new Descriptor(snapshotDirectory, sstable.getKeyspaceName(), sstable.getColumnFamilyName(), sstable.descriptor.generation, sstable.descriptor.formatType);
            Set<Component> components = snapshotSSTables.get(snapshotSSTable);
            Assert.assertNotNull(components);
            Assert.assertTrue(components.containsAll(sasiComponents));
            for (Component c : components) {
                Path componentPath = Paths.get(sstable.descriptor + "-" + c.name);
                long componentSize = Files.size(componentPath);
                if (Component.Type.fromRepresentation(c.name) == Component.Type.SECONDARY_INDEX)
                    indexSize += componentSize;
                else
                    tableSize += componentSize;
            }
        }
        TableSnapshot details = store.listSnapshots().get(snapshotName);
        // check that SASI components are included in the computation of snapshot size
        Assert.assertEquals(details.computeTrueSizeBytes(), tableSize + indexSize);
    } finally {
        store.clearSnapshot(snapshotName);
    }
}

Example 4 with SnapshotManifest

use of org.apache.cassandra.service.snapshot.SnapshotManifest in project cassandra by apache.

the class ColumnFamilyStore method createSnapshot.

protected TableSnapshot createSnapshot(String tag, boolean ephemeral, DurationSpec ttl, Set<SSTableReader> sstables, Instant creationTime) {
    Set<File> snapshotDirs = sstables.stream().map(s -> Directories.getSnapshotDirectory(s.descriptor, tag).toAbsolute()).filter(// Remove secondary index subdirectory
    dir -> !Directories.isSecondaryIndexFolder(dir)).collect(Collectors.toCollection(HashSet::new));
    // Create and write snapshot manifest
    SnapshotManifest manifest = new SnapshotManifest(mapToDataFilenames(sstables), ttl, creationTime);
    File manifestFile = getDirectories().getSnapshotManifestFile(tag);
    writeSnapshotManifest(manifest, manifestFile);
    // manifest may create empty snapshot dir
    snapshotDirs.add(manifestFile.parent().toAbsolute());
    // Write snapshot schema
    if (!SchemaConstants.isLocalSystemKeyspace(metadata.keyspace) && !SchemaConstants.isReplicatedSystemKeyspace(metadata.keyspace)) {
        File schemaFile = getDirectories().getSnapshotSchemaFile(tag);
        writeSnapshotSchema(schemaFile);
        // schema may create empty snapshot dir
        snapshotDirs.add(schemaFile.parent().toAbsolute());
    }
    // Maybe create ephemeral marker
    if (ephemeral) {
        File ephemeralSnapshotMarker = getDirectories().getNewEphemeralSnapshotMarkerFile(tag);
        createEphemeralSnapshotMarkerFile(tag, ephemeralSnapshotMarker);
        // marker may create empty snapshot dir
        snapshotDirs.add(ephemeralSnapshotMarker.parent().toAbsolute());
    }
    TableSnapshot snapshot = new TableSnapshot(metadata.keyspace, metadata.name, tag, manifest.createdAt, manifest.expiresAt, snapshotDirs, directories::getTrueAllocatedSizeIn);
    StorageService.instance.addSnapshot(snapshot);
    return snapshot;
}

Example 5 with SnapshotManifest

use of org.apache.cassandra.service.snapshot.SnapshotManifest in project cassandra by apache.

the class DirectoriesTest method createFakeSnapshot.

public FakeSnapshot createFakeSnapshot(TableMetadata table, String tag, boolean createManifest) throws IOException {
    File tableDir = cfDir(table);
    tableDir.tryCreateDirectories();
    File snapshotDir = new File(tableDir, Directories.SNAPSHOT_SUBDIR + File.pathSeparator() + tag);
    snapshotDir.tryCreateDirectories();
    Descriptor sstableDesc = new Descriptor(snapshotDir, KS, table.name, 1, SSTableFormat.Type.BIG);
    createFakeSSTable(sstableDesc);
    SnapshotManifest manifest = null;
    if (createManifest) {
        File manifestFile = Directories.getSnapshotManifestFile(snapshotDir);
        manifest = new SnapshotManifest(Collections.singletonList(sstableDesc.filenameFor(Component.DATA)), new DurationSpec("1m"), Instant.now());
        manifest.serializeToJsonFile(manifestFile);
    }
    return new FakeSnapshot(table, tag, snapshotDir, manifest);
}