Examples with Component org.apache.cassandra.io.sstable.Component used on opensource projects

Example 11 with Component

use of org.apache.cassandra.io.sstable.Component in project cassandra by apache.

the class SASIIndexTest method testSASIComponentsAddedToSnapshot.

@Test
public void testSASIComponentsAddedToSnapshot() throws Throwable {
    String snapshotName = "sasi_test";
    Map<String, Pair<String, Integer>> data = new HashMap<>();
    Random r = new Random();
    for (int i = 0; i < 100; i++) data.put(UUID.randomUUID().toString(), Pair.create(UUID.randomUUID().toString(), r.nextInt()));
    ColumnFamilyStore store = loadData(data, true);
    store.forceMajorCompaction();
    Set<SSTableReader> ssTableReaders = store.getLiveSSTables();
    Set<Component> sasiComponents = new HashSet<>();
    for (Index index : store.indexManager.listIndexes()) if (index instanceof SASIIndex)
        sasiComponents.add(((SASIIndex) index).getIndex().getComponent());
    Assert.assertFalse(sasiComponents.isEmpty());
    try {
        store.snapshot(snapshotName);
        SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(store.getDirectories().getSnapshotManifestFile(snapshotName));
        Assert.assertFalse(ssTableReaders.isEmpty());
        Assert.assertFalse(manifest.files.isEmpty());
        Assert.assertEquals(ssTableReaders.size(), manifest.files.size());
        Map<Descriptor, Set<Component>> snapshotSSTables = store.getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots(snapshotName).list();
        long indexSize = 0;
        long tableSize = 0;
        for (SSTableReader sstable : ssTableReaders) {
            File snapshotDirectory = Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
            Descriptor snapshotSSTable = new Descriptor(snapshotDirectory, sstable.getKeyspaceName(), sstable.getColumnFamilyName(), sstable.descriptor.generation, sstable.descriptor.formatType);
            Set<Component> components = snapshotSSTables.get(snapshotSSTable);
            Assert.assertNotNull(components);
            Assert.assertTrue(components.containsAll(sasiComponents));
            for (Component c : components) {
                Path componentPath = Paths.get(sstable.descriptor + "-" + c.name);
                long componentSize = Files.size(componentPath);
                if (Component.Type.fromRepresentation(c.name) == Component.Type.SECONDARY_INDEX)
                    indexSize += componentSize;
                else
                    tableSize += componentSize;
            }
        }
        TableSnapshot details = store.listSnapshots().get(snapshotName);
        // check that SASI components are included in the computation of snapshot size
        Assert.assertEquals(details.computeTrueSizeBytes(), tableSize + indexSize);
    } finally {
        store.clearSnapshot(snapshotName);
    }
}

Example 12 with Component

use of org.apache.cassandra.io.sstable.Component in project cassandra by apache.

the class LogRecord method getExistingFiles.

/**
 * absoluteFilePaths contains full file parts up to (but excluding) the component name
 *
 * This method finds all files on disk beginning with any of the paths in absoluteFilePaths
 *
 * @return a map from absoluteFilePath to actual file on disk.
 */
public static Map<String, List<File>> getExistingFiles(Set<String> absoluteFilePaths) {
    Map<String, List<File>> fileMap = new HashMap<>();
    Map<File, TreeSet<String>> dirToFileNamePrefix = new HashMap<>();
    for (String absolutePath : absoluteFilePaths) {
        Path fullPath = new File(absolutePath).toPath();
        Path path = fullPath.getParent();
        if (path != null)
            dirToFileNamePrefix.computeIfAbsent(new File(path), (k) -> new TreeSet<>()).add(fullPath.getFileName().toString());
    }
    BiPredicate<File, String> ff = (dir, name) -> {
        TreeSet<String> dirSet = dirToFileNamePrefix.get(dir);
        // if the set contains a prefix of the current file name, the file name we have here should sort directly
        // after the prefix in the tree set, which means we can use 'floor' to get the prefix (returns the largest
        // of the smaller strings in the set). Also note that the prefixes always end with '-' which means we won't
        // have "xy-1111-Data.db".startsWith("xy-11") below (we'd get "xy-1111-Data.db".startsWith("xy-11-"))
        String baseName = dirSet.floor(name);
        if (baseName != null && name.startsWith(baseName)) {
            String absolutePath = new File(dir, baseName).path();
            fileMap.computeIfAbsent(absolutePath, k -> new ArrayList<>()).add(new File(dir, name));
        }
        return false;
    };
    // populate the file map:
    for (File f : dirToFileNamePrefix.keySet()) f.tryList(ff);
    return fileMap;
}

Example 13 with Component

use of org.apache.cassandra.io.sstable.Component in project cassandra by apache.

the class CassandraEntireSSTableStreamWriter method write.

/**
 * Stream the entire file to given channel.
 * <p>
 * TODO: this currently requires a companion thread, but could be performed entirely asynchronously
 * @param out where this writes data to
 * @throws IOException on any I/O error
 */
public void write(StreamingDataOutputPlus out) throws IOException {
    long totalSize = manifest.totalSize();
    logger.debug("[Stream #{}] Start streaming sstable {} to {}, repairedAt = {}, totalSize = {}", session.planId(), sstable.getFilename(), session.peer, sstable.getSSTableMetadata().repairedAt, prettyPrintMemory(totalSize));
    long progress = 0L;
    for (Component component : manifest.components()) {
        // Total Length to transmit for this file
        long length = manifest.sizeOf(component);
        // tracks write progress
        logger.debug("[Stream #{}] Streaming {}.{} gen {} component {} size {}", session.planId(), sstable.getKeyspaceName(), sstable.getColumnFamilyName(), sstable.descriptor.generation, component, prettyPrintMemory(length));
        // this is closed after the file is transferred by AsyncChannelOutputPlus
        @SuppressWarnings("resource") FileChannel channel = context.channel(sstable.descriptor, component, length);
        long bytesWritten = out.writeFileToChannel(channel, limiter);
        progress += bytesWritten;
        session.progress(sstable.descriptor.filenameFor(component), ProgressInfo.Direction.OUT, bytesWritten, length);
        logger.debug("[Stream #{}] Finished streaming {}.{} gen {} component {} to {}, xfered = {}, length = {}, totalSize = {}", session.planId(), sstable.getKeyspaceName(), sstable.getColumnFamilyName(), sstable.descriptor.generation, component, session.peer, prettyPrintMemory(bytesWritten), prettyPrintMemory(length), prettyPrintMemory(totalSize));
    }
    out.flush();
    logger.debug("[Stream #{}] Finished streaming sstable {} to {}, xfered = {}, totalSize = {}", session.planId(), sstable.getFilename(), session.peer, prettyPrintMemory(progress), prettyPrintMemory(totalSize));
}

Example 14 with Component

use of org.apache.cassandra.io.sstable.Component in project cassandra by apache.

the class ComponentContext method create.

public static ComponentContext create(Descriptor descriptor) {
    Map<Component, File> hardLinks = new HashMap<>(1);
    for (Component component : MUTABLE_COMPONENTS) {
        File file = new File(descriptor.filenameFor(component));
        if (!file.exists())
            continue;
        File hardlink = new File(descriptor.tmpFilenameForStreaming(component));
        FileUtils.createHardLink(file, hardlink);
        hardLinks.put(component, hardlink);
    }
    return new ComponentContext(hardLinks, ComponentManifest.create(descriptor));
}

Example 15 with Component

use of org.apache.cassandra.io.sstable.Component in project cassandra by apache.

the class ComponentManifest method create.

@VisibleForTesting
public static ComponentManifest create(Descriptor descriptor) {
    LinkedHashMap<Component, Long> components = new LinkedHashMap<>(STREAM_COMPONENTS.size());
    for (Component component : STREAM_COMPONENTS) {
        File file = new File(descriptor.filenameFor(component));
        if (!file.exists())
            continue;
        components.put(component, file.length());
    }
    return new ComponentManifest(components);
}