Examples with SSTableLoader org.apache.cassandra.io.sstable.SSTableLoader used on opensource projects

Example 6 with SSTableLoader

use of org.apache.cassandra.io.sstable.SSTableLoader in project cassandra by apache.

the class LongStreamingTest method testCompressedStream.

@Test
public void testCompressedStream() throws InvalidRequestException, IOException, ExecutionException, InterruptedException {
    String KS = "cql_keyspace";
    String TABLE = "table1";
    File tempdir = Files.createTempDir();
    File dataDir = new File(tempdir.getAbsolutePath() + File.separator + KS + File.separator + TABLE);
    assert dataDir.mkdirs();
    String schema = "CREATE TABLE cql_keyspace.table1 (" + "  k int PRIMARY KEY," + "  v1 text," + "  v2 int" + // with compression = {};";
    ");";
    String insert = "INSERT INTO cql_keyspace.table1 (k, v1, v2) VALUES (?, ?, ?)";
    CQLSSTableWriter writer = CQLSSTableWriter.builder().sorted().inDirectory(dataDir).forTable(schema).using(insert).build();
    long start = System.nanoTime();
    for (int i = 0; i < 10_000_000; i++) writer.addRow(i, "test1", 24);
    writer.close();
    System.err.println(String.format("Writer finished after %d seconds....", TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start)));
    File[] dataFiles = dataDir.listFiles((dir, name) -> name.endsWith("-Data.db"));
    long dataSize = 0l;
    for (File file : dataFiles) {
        System.err.println("File : " + file.getAbsolutePath());
        dataSize += file.length();
    }
    SSTableLoader loader = new SSTableLoader(dataDir, new SSTableLoader.Client() {

        private String ks;

        public void init(String keyspace) {
            for (Range<Token> range : StorageService.instance.getLocalRanges("cql_keyspace")) addRangeForEndpoint(range, FBUtilities.getBroadcastAddress());
            this.ks = keyspace;
        }

        public TableMetadataRef getTableMetadata(String cfName) {
            return Schema.instance.getTableMetadataRef(ks, cfName);
        }
    }, new OutputHandler.SystemOutput(false, false));
    start = System.nanoTime();
    loader.stream().get();
    long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
    System.err.println(String.format("Finished Streaming in %.2f seconds: %.2f Mb/sec", millis / 1000d, (dataSize / (1 << 20) / (millis / 1000d)) * 8));
    //Stream again
    loader = new SSTableLoader(dataDir, new SSTableLoader.Client() {

        private String ks;

        public void init(String keyspace) {
            for (Range<Token> range : StorageService.instance.getLocalRanges("cql_keyspace")) addRangeForEndpoint(range, FBUtilities.getBroadcastAddress());
            this.ks = keyspace;
        }

        public TableMetadataRef getTableMetadata(String cfName) {
            return Schema.instance.getTableMetadataRef(ks, cfName);
        }
    }, new OutputHandler.SystemOutput(false, false));
    start = System.nanoTime();
    loader.stream().get();
    millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
    System.err.println(String.format("Finished Streaming in %.2f seconds: %.2f Mb/sec", millis / 1000d, (dataSize / (1 << 20) / (millis / 1000d)) * 8));
    //Compact them both
    start = System.nanoTime();
    Keyspace.open(KS).getColumnFamilyStore(TABLE).forceMajorCompaction();
    millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
    System.err.println(String.format("Finished Compacting in %.2f seconds: %.2f Mb/sec", millis / 1000d, (dataSize * 2 / (1 << 20) / (millis / 1000d)) * 8));
    UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM cql_keyspace.table1 limit 100;");
    assertEquals(100, rs.size());
}