Examples with SSTableReader org.apache.cassandra.io.sstable.SSTableReader used on opensource projects

Example 26 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project eiger by wlloyd.

the class LongCompactionSpeedTest method testCompaction.

protected void testCompaction(int sstableCount, int rowsPerSSTable, int colsPerRow) throws Exception {
    CompactionManager.instance.disableAutoCompaction();
    Table table = Table.open(TABLE1);
    ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
    ArrayList<SSTableReader> sstables = new ArrayList<SSTableReader>();
    for (int k = 0; k < sstableCount; k++) {
        SortedMap<String, ColumnFamily> rows = new TreeMap<String, ColumnFamily>();
        for (int j = 0; j < rowsPerSSTable; j++) {
            String key = String.valueOf(j);
            IColumn[] cols = new IColumn[colsPerRow];
            for (int i = 0; i < colsPerRow; i++) {
                // last sstable has highest timestamps
                cols[i] = Util.column(String.valueOf(i), String.valueOf(i), k);
            }
            rows.put(key, SSTableUtils.createCF(Long.MIN_VALUE, Integer.MIN_VALUE, cols));
        }
        SSTableReader sstable = SSTableUtils.prepare().write(rows);
        sstables.add(sstable);
        store.addSSTable(sstable);
    }
    // give garbage collection a bit of time to catch up
    Thread.sleep(1000);
    long start = System.currentTimeMillis();
    final int gcBefore = (int) (System.currentTimeMillis() / 1000) - Schema.instance.getCFMetaData(TABLE1, "Standard1").getGcGraceSeconds();
    new CompactionTask(store, sstables, gcBefore).execute(null);
    System.out.println(String.format("%s: sstables=%d rowsper=%d colsper=%d: %d ms", this.getClass().getName(), sstableCount, rowsPerSSTable, colsPerRow, System.currentTimeMillis() - start));
}

Example 27 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project eiger by wlloyd.

the class SSTableExportTest method testRoundTripStandardCf.

@Test
public void testRoundTripStandardCf() throws IOException, ParseException {
    File tempSS = tempSSTableFile("Keyspace1", "Standard1");
    ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Standard1");
    SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
    // Add rowA
    cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("name")), ByteBufferUtil.bytes("val"), System.currentTimeMillis());
    writer.append(Util.dk("rowA"), cfamily);
    cfamily.clear();
    // Add rowExclude
    cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("name")), ByteBufferUtil.bytes("val"), System.currentTimeMillis());
    writer.append(Util.dk("rowExclude"), cfamily);
    cfamily.clear();
    SSTableReader reader = writer.closeAndOpenReader();
    // Export to JSON and verify
    File tempJson = File.createTempFile("Standard1", ".json");
    SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[] { asHex("rowExclude") });
    // Import JSON to another SSTable file
    File tempSS2 = tempSSTableFile("Keyspace1", "Standard1");
    SSTableImport.importJson(tempJson.getPath(), "Keyspace1", "Standard1", tempSS2.getPath());
    reader = SSTableReader.open(Descriptor.fromFilename(tempSS2.getPath()));
    QueryFilter qf = QueryFilter.getNamesFilter(Util.dk("rowA"), new QueryPath("Standard1", null, null), ByteBufferUtil.bytes("name"));
    ColumnFamily cf = qf.getSSTableColumnIterator(reader).getColumnFamily();
    assertTrue(cf != null);
    assertTrue(cf.getColumn(ByteBufferUtil.bytes("name")).value().equals(hexToBytes("76616c")));
    qf = QueryFilter.getNamesFilter(Util.dk("rowExclude"), new QueryPath("Standard1", null, null), ByteBufferUtil.bytes("name"));
    cf = qf.getSSTableColumnIterator(reader).getColumnFamily();
    assert cf == null;
}

Example 28 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project eiger by wlloyd.

the class SSTableExportTest method testExportSuperCf.

@Test
public void testExportSuperCf() throws IOException {
    File tempSS = tempSSTableFile("Keyspace1", "Super4");
    ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Super4");
    SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
    // Add rowA
    cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superA"), ByteBufferUtil.bytes("colA")), ByteBufferUtil.bytes("valA"), System.currentTimeMillis());
    writer.append(Util.dk("rowA"), cfamily);
    cfamily.clear();
    // Add rowB
    cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superB"), ByteBufferUtil.bytes("colB")), ByteBufferUtil.bytes("valB"), System.currentTimeMillis());
    writer.append(Util.dk("rowB"), cfamily);
    cfamily.clear();
    // Add rowExclude
    cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superX"), ByteBufferUtil.bytes("colX")), ByteBufferUtil.bytes("valX"), System.currentTimeMillis());
    writer.append(Util.dk("rowExclude"), cfamily);
    cfamily.clear();
    SSTableReader reader = writer.closeAndOpenReader();
    // Export to JSON and verify
    File tempJson = File.createTempFile("Super4", ".json");
    SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[] { asHex("rowExclude") });
    JSONObject json = (JSONObject) JSONValue.parse(new FileReader(tempJson));
    JSONObject rowA = (JSONObject) json.get(asHex("rowA"));
    JSONObject superA = (JSONObject) rowA.get(cfamily.getComparator().getString(ByteBufferUtil.bytes("superA")));
    JSONArray subColumns = (JSONArray) superA.get("subColumns");
    JSONArray colA = (JSONArray) subColumns.get(0);
    JSONObject rowExclude = (JSONObject) json.get(asHex("rowExclude"));
    assert hexToBytes((String) colA.get(1)).equals(ByteBufferUtil.bytes("valA"));
    assert colA.size() == 3;
    assert rowExclude == null;
}

Example 29 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project eiger by wlloyd.

the class SSTableExportTest method testExportSimpleCf.

@Test
public void testExportSimpleCf() throws IOException {
    File tempSS = tempSSTableFile("Keyspace1", "Standard1");
    ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Standard1");
    SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
    // live for 42 seconds
    int nowInSec = (int) (System.currentTimeMillis() / 1000) + 42;
    // Add rowA
    cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colA")), ByteBufferUtil.bytes("valA"), System.currentTimeMillis());
    cfamily.addColumn(null, new ExpiringColumn(ByteBufferUtil.bytes("colExp"), ByteBufferUtil.bytes("valExp"), System.currentTimeMillis(), 42, nowInSec));
    writer.append(Util.dk("rowA"), cfamily);
    cfamily.clear();
    // Add rowB
    cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colB")), ByteBufferUtil.bytes("valB"), System.currentTimeMillis());
    writer.append(Util.dk("rowB"), cfamily);
    cfamily.clear();
    // Add rowExclude
    cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colX")), ByteBufferUtil.bytes("valX"), System.currentTimeMillis());
    writer.append(Util.dk("rowExclude"), cfamily);
    cfamily.clear();
    SSTableReader reader = writer.closeAndOpenReader();
    // Export to JSON and verify
    File tempJson = File.createTempFile("Standard1", ".json");
    SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[] { asHex("rowExclude") });
    JSONObject json = (JSONObject) JSONValue.parse(new FileReader(tempJson));
    JSONArray rowA = (JSONArray) json.get(asHex("rowA"));
    JSONArray colA = (JSONArray) rowA.get(0);
    assert hexToBytes((String) colA.get(1)).equals(ByteBufferUtil.bytes("valA"));
    JSONArray colExp = (JSONArray) rowA.get(1);
    assert ((Long) colExp.get(4)) == 42;
    assert ((Long) colExp.get(5)) == nowInSec;
    JSONArray rowB = (JSONArray) json.get(asHex("rowB"));
    JSONArray colB = (JSONArray) rowB.get(0);
    assert colB.size() == 3;
    JSONArray rowExclude = (JSONArray) json.get(asHex("rowExclude"));
    assert rowExclude == null;
}

Example 30 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project eiger by wlloyd.

the class SecondaryIndex method buildIndexBlocking.

/**
 * Builds the index using the data in the underlying CFS
 * Blocks till it's complete
 */
protected void buildIndexBlocking() throws IOException {
    logger.info(String.format("Submitting index build of %s for data in %s", getIndexName(), StringUtils.join(baseCfs.getSSTables(), ", ")));
    SortedSet<ByteBuffer> columnNames = new TreeSet<ByteBuffer>();
    for (ColumnDefinition cdef : columnDefs) columnNames.add(cdef.name);
    Collection<SSTableReader> sstables = baseCfs.markCurrentSSTablesReferenced();
    SecondaryIndexBuilder builder = new SecondaryIndexBuilder(baseCfs, columnNames, new ReducingKeyIterator(sstables));
    Future<?> future = CompactionManager.instance.submitIndexBuild(builder);
    try {
        future.get();
        forceBlockingFlush();
        // Mark all indexed columns as built
        if (this instanceof PerRowSecondaryIndex) {
            for (ByteBuffer columnName : columnNames) SystemTable.setIndexBuilt(baseCfs.table.name, getIndexName() + ByteBufferUtil.string(columnName));
        } else {
            SystemTable.setIndexBuilt(baseCfs.table.name, getIndexName());
        }
    } catch (InterruptedException e) {
        throw new AssertionError(e);
    } catch (ExecutionException e) {
        throw new IOException(e);
    } finally {
        SSTableReader.releaseReferences(sstables);
    }
    logger.info("Index build of " + getIndexName() + " complete");
}