Examples with SSTableSplitter org.apache.cassandra.db.compaction.SSTableSplitter used on opensource projects

Example 1 with SSTableSplitter

use of org.apache.cassandra.db.compaction.SSTableSplitter in project cassandra by apache.

the class StandaloneSplitter method main.

public static void main(String[] args) {
    Options options = Options.parseArgs(args);
    Util.initDatabaseDescriptor();
    try {
        // load keyspace descriptions.
        Schema.instance.loadFromDisk(false);
        String ksName = null;
        String cfName = null;
        Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
        for (String filename : options.filenames) {
            File file = new File(filename);
            if (!file.exists()) {
                System.out.println("Skipping inexisting file " + file);
                continue;
            }
            Descriptor desc = SSTable.tryDescriptorFromFilename(file);
            if (desc == null) {
                System.out.println("Skipping non sstable file " + file);
                continue;
            }
            if (ksName == null)
                ksName = desc.ksname;
            else if (!ksName.equals(desc.ksname))
                throw new IllegalArgumentException("All sstables must be part of the same keyspace");
            if (cfName == null)
                cfName = desc.cfname;
            else if (!cfName.equals(desc.cfname))
                throw new IllegalArgumentException("All sstables must be part of the same table");
            Set<Component> components = new HashSet<Component>(Arrays.asList(new Component[] { Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.COMPRESSION_INFO, Component.STATS }));
            Iterator<Component> iter = components.iterator();
            while (iter.hasNext()) {
                Component component = iter.next();
                if (!(new File(desc.filenameFor(component)).exists()))
                    iter.remove();
            }
            parsedFilenames.put(desc, components);
        }
        if (ksName == null || cfName == null) {
            System.err.println("No valid sstables to split");
            System.exit(1);
        }
        // Do not load sstables since they might be broken
        Keyspace keyspace = Keyspace.openWithoutSSTables(ksName);
        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName);
        String snapshotName = "pre-split-" + System.currentTimeMillis();
        List<SSTableReader> sstables = new ArrayList<>();
        for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) {
            try {
                SSTableReader sstable = SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs);
                if (!isSSTableLargerEnough(sstable, options.sizeInMB)) {
                    System.out.println(String.format("Skipping %s: it's size (%.3f MB) is less than the split size (%d MB)", sstable.getFilename(), ((sstable.onDiskLength() * 1.0d) / 1024L) / 1024L, options.sizeInMB));
                    continue;
                }
                sstables.add(sstable);
                if (options.snapshot) {
                    File snapshotDirectory = Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
                    sstable.createLinks(snapshotDirectory.getPath());
                }
            } catch (Exception e) {
                JVMStabilityInspector.inspectThrowable(e);
                System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
                if (options.debug)
                    e.printStackTrace(System.err);
            }
        }
        if (sstables.isEmpty()) {
            System.out.println("No sstables needed splitting.");
            System.exit(0);
        }
        if (options.snapshot)
            System.out.println(String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));
        for (SSTableReader sstable : sstables) {
            try (LifecycleTransaction transaction = LifecycleTransaction.offline(OperationType.UNKNOWN, sstable)) {
                new SSTableSplitter(cfs, transaction, options.sizeInMB).split();
            } catch (Exception e) {
                System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
                if (options.debug)
                    e.printStackTrace(System.err);
                sstable.selfRef().release();
            }
        }
        CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
        LifecycleTransaction.waitForDeletions();
        // We need that to stop non daemonized threads
        System.exit(0);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        if (options.debug)
            e.printStackTrace(System.err);
        System.exit(1);
    }
}

Example 2 with SSTableSplitter

use of org.apache.cassandra.db.compaction.SSTableSplitter in project cassandra by apache.

the class SSTableRewriterTest method testSSTableSplit.

@Test
public void testSSTableSplit() throws InterruptedException {
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);
    cfs.disableAutoCompaction();
    SSTableReader s = writeFile(cfs, 1000);
    try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.UNKNOWN, s)) {
        SSTableSplitter splitter = new SSTableSplitter(cfs, txn, 10);
        splitter.split();
        assertFileCounts(s.descriptor.directory.list());
        LifecycleTransaction.waitForDeletions();
        for (File f : s.descriptor.directory.listFiles()) {
            // we need to clear out the data dir, otherwise tests running after this breaks
            FileUtils.deleteRecursive(f);
        }
    }
    truncate(cfs);
}