Examples with DataFrame com.questdb.cairo.sql.DataFrame used on opensource projects

Example 1 with DataFrame

use of com.questdb.cairo.sql.DataFrame in project questdb by bluestreak01.

the class FullTableFrameCursorFactoryTest method testFactory.

@Test
public void testFactory() throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        final int N = 100;
        // separate two symbol columns with primitive. It will make problems apparent if index does not shift correctly
        try (TableModel model = new TableModel(configuration, "x", PartitionBy.DAY).col("a", ColumnType.STRING).col("b", ColumnType.SYMBOL).indexed(true, N / 4).col("i", ColumnType.INT).col("c", ColumnType.SYMBOL).indexed(true, N / 4).timestamp()) {
            CairoTestUtils.create(model);
        }
        final Rnd rnd = new Rnd();
        final String[] symbols = new String[N];
        final int M = 1000;
        final long increment = 1000000 * 60L * 10;
        for (int i = 0; i < N; i++) {
            symbols[i] = rnd.nextChars(8).toString();
        }
        // prepare the data
        long timestamp = 0;
        try (TableWriter writer = new TableWriter(configuration, "x")) {
            for (int i = 0; i < M; i++) {
                TableWriter.Row row = writer.newRow(timestamp += increment);
                row.putStr(0, rnd.nextChars(20));
                row.putSym(1, symbols[rnd.nextPositiveInt() % N]);
                row.putInt(2, rnd.nextInt());
                row.putSym(3, symbols[rnd.nextPositiveInt() % N]);
                row.append();
            }
            writer.commit();
        }
        try (Engine engine = new Engine(configuration)) {
            FullTableFrameCursorFactory factory = new FullTableFrameCursorFactory(engine, "x");
            long count = 0;
            try (DataFrameCursor cursor = factory.getCursor()) {
                while (cursor.hasNext()) {
                    DataFrame frame = cursor.next();
                    count += frame.getRowHi() - frame.getRowLo();
                }
            }
            Assert.assertEquals(0, engine.getBusyReaderCount());
            Assert.assertEquals(M, count);
        }
    });
}

Example 2 with DataFrame

use of com.questdb.cairo.sql.DataFrame in project questdb by bluestreak01.

the class FullTableFrameCursorTest method assertSymbolFoundInIndex.

private void assertSymbolFoundInIndex(FullTableFrameCursor cursor, TableReaderRecord record, int columnIndex, int M) {
    // SymbolTable is table at table scope, so it will be the same for every
    // data frame here. Get its instance outside of data frame loop.
    SymbolTable symbolTable = cursor.getSymbolTable(columnIndex);
    long count = 0;
    while (cursor.hasNext()) {
        DataFrame frame = cursor.next();
        record.jumpTo(frame.getPartitionIndex(), frame.getRowLo());
        final long limit = frame.getRowHi();
        // BitmapIndex is always at data frame scope, each table can have more than one.
        // we have to get BitmapIndexReader instance once for each frame.
        BitmapIndexReader indexReader = frame.getBitmapIndexReader(columnIndex);
        // because out Symbol column 0 is indexed, frame has to have index.
        Assert.assertNotNull(indexReader);
        // Iterate data frame and advance record by incrementing "recordIndex"
        long recordIndex;
        while ((recordIndex = record.getRecordIndex()) < limit) {
            CharSequence sym = record.getSym(columnIndex);
            // Assert that index cursor contains offset of current row
            boolean offsetFound = false;
            long target = record.getRecordIndex();
            // Get index cursor for each symbol in data frame
            RowCursor ic = indexReader.getCursor(symbolTable.getQuick(sym) + 1, limit - 1);
            while (ic.hasNext()) {
                if (ic.next() == target) {
                    offsetFound = true;
                    break;
                }
            }
            Assert.assertTrue(offsetFound);
            record.setRecordIndex(recordIndex + 1);
            count++;
        }
    }
    // assert that we read entire table
    Assert.assertEquals(M, count);
}

Example 3 with DataFrame

use of com.questdb.cairo.sql.DataFrame in project questdb by bluestreak01.

the class FullTableFrameCursorTest method assertNoIndex.

private void assertNoIndex(FullTableFrameCursor cursor) {
    while (cursor.hasNext()) {
        DataFrame frame = cursor.next();
        try {
            frame.getBitmapIndexReader(4);
            Assert.fail();
        } catch (CairoException e) {
            Assert.assertTrue(Chars.contains(e.getMessage(), "Not indexed"));
        }
    }
}

Example 4 with DataFrame

use of com.questdb.cairo.sql.DataFrame in project questdb by bluestreak01.

the class FullTableFrameCursorTest method assertIndexRowsMatchSymbol.

static void assertIndexRowsMatchSymbol(DataFrameCursor cursor, TableReaderRecord record, int columnIndex, long expecteRowCount) {
    // SymbolTable is table at table scope, so it will be the same for every
    // data frame here. Get its instance outside of data frame loop.
    SymbolTable symbolTable = cursor.getSymbolTable(columnIndex);
    long rowCount = 0;
    while (cursor.hasNext()) {
        DataFrame frame = cursor.next();
        record.jumpTo(frame.getPartitionIndex(), frame.getRowLo());
        final long limit = frame.getRowHi();
        // BitmapIndex is always at data frame scope, each table can have more than one.
        // we have to get BitmapIndexReader instance once for each frame.
        BitmapIndexReader indexReader = frame.getBitmapIndexReader(columnIndex);
        // because out Symbol column 0 is indexed, frame has to have index.
        Assert.assertNotNull(indexReader);
        int keyCount = indexReader.getKeyCount();
        for (int i = 0; i < keyCount; i++) {
            RowCursor ic = indexReader.getCursor(i, limit - 1);
            CharSequence expected = symbolTable.value(i - 1);
            while (ic.hasNext()) {
                record.setRecordIndex(ic.next());
                TestUtils.assertEquals(expected, record.getSym(columnIndex));
                rowCount++;
            }
        }
    }
    Assert.assertEquals(expecteRowCount, rowCount);
}

Example 5 with DataFrame

use of com.questdb.cairo.sql.DataFrame in project questdb by bluestreak01.

the class IntervalFrameCursorTest method collectTimestamps.

private void collectTimestamps(DataFrameCursor cursor, TableReaderRecord record, CharSink sink) {
    int timestampIndex = cursor.getReader().getMetadata().getTimestampIndex();
    while (cursor.hasNext()) {
        DataFrame frame = cursor.next();
        record.jumpTo(frame.getPartitionIndex(), frame.getRowLo());
        long limit = frame.getRowHi();
        long recordIndex;
        while ((recordIndex = record.getRecordIndex()) < limit) {
            sink.putISODate(record.getDate(timestampIndex)).put('\n');
            record.setRecordIndex(recordIndex + 1);
        }
    }
}