Examples with RecordCursor com.questdb.common.RecordCursor used on opensource projects

Example 21 with RecordCursor

use of com.questdb.common.RecordCursor in project questdb by bluestreak01.

the class TableReadFailTest method testReloadTimeout.

@Test
public void testReloadTimeout() throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        try (TableModel model = new TableModel(configuration, "x", PartitionBy.NONE).col("a", ColumnType.INT).col("b", ColumnType.LONG).timestamp()) {
            CairoTestUtils.create(model);
        }
        try (Path path = new Path();
            TableReader reader = new TableReader(configuration, "x");
            ReadWriteMemory mem = new ReadWriteMemory()) {
            final Rnd rnd = new Rnd();
            final int N = 1000;
            // home path at txn file
            path.of(configuration.getRoot()).concat("x").concat(TableUtils.TXN_FILE_NAME).$();
            try (TableWriter w = new TableWriter(configuration, "x")) {
                for (int i = 0; i < N; i++) {
                    TableWriter.Row r = w.newRow(0);
                    r.putInt(0, rnd.nextInt());
                    r.putLong(1, rnd.nextLong());
                    r.append();
                }
                w.commit();
            }
            Assert.assertTrue(reader.reload());
            RecordCursor cursor = reader.getCursor();
            rnd.reset();
            int count = 0;
            while (cursor.hasNext()) {
                Record r = cursor.next();
                Assert.assertEquals(rnd.nextInt(), r.getInt(0));
                Assert.assertEquals(rnd.nextLong(), r.getLong(1));
                count++;
            }
            Assert.assertEquals(N, count);
            mem.of(configuration.getFilesFacade(), path, configuration.getFilesFacade().getPageSize());
            // keep txn file parameters
            long offset = configuration.getFilesFacade().length(mem.getFd());
            long txn = mem.getLong(TableUtils.TX_OFFSET_TXN);
            // corrupt the txn file
            mem.jumpTo(TableUtils.TX_OFFSET_TXN);
            mem.putLong(123);
            mem.jumpTo(offset);
            mem.close();
            // this should time out
            try {
                reader.reload();
                Assert.fail();
            } catch (CairoException e) {
                TestUtils.assertContains(e.getMessage(), "timeout");
            }
            // restore txn file to its former glory
            mem.of(configuration.getFilesFacade(), path, configuration.getFilesFacade().getPageSize());
            mem.jumpTo(TableUtils.TX_OFFSET_TXN);
            mem.putLong(txn);
            mem.jumpTo(offset);
            mem.close();
            mem.close();
            // make sure reload functions correctly
            Assert.assertFalse(reader.reload());
            try (TableWriter w = new TableWriter(configuration, "x")) {
                // add more data
                for (int i = 0; i < N; i++) {
                    TableWriter.Row r = w.newRow(0);
                    r.putInt(0, rnd.nextInt());
                    r.putLong(1, rnd.nextLong());
                    r.append();
                }
                w.commit();
            }
            // does positive reload work?
            Assert.assertTrue(reader.reload());
            // can reader still see correct data?
            cursor = reader.getCursor();
            rnd.reset();
            count = 0;
            while (cursor.hasNext()) {
                Record r = cursor.next();
                Assert.assertEquals(rnd.nextInt(), r.getInt(0));
                Assert.assertEquals(rnd.nextLong(), r.getLong(1));
                count++;
            }
            Assert.assertEquals(2 * N, count);
        }
    });
}

Example 22 with RecordCursor

use of com.questdb.common.RecordCursor in project questdb by bluestreak01.

the class TableReaderRecordCursorFactoryTest method testFactory.

@Test
public void testFactory() throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        final int N = 100;
        // separate two symbol columns with primitive. It will make problems apparent if index does not shift correctly
        try (TableModel model = new TableModel(configuration, "x", PartitionBy.DAY).col("a", ColumnType.STRING).col("b", ColumnType.SYMBOL).indexed(true, N / 4).col("i", ColumnType.INT).col("c", ColumnType.SYMBOL).indexed(true, N / 4).timestamp()) {
            CairoTestUtils.create(model);
        }
        final Rnd rnd = new Rnd();
        final String[] symbols = new String[N];
        final int M = 1000;
        final long increment = 1000000 * 60L * 10;
        for (int i = 0; i < N; i++) {
            symbols[i] = rnd.nextChars(8).toString();
        }
        rnd.reset();
        // prepare the data
        long timestamp = 0;
        try (TableWriter writer = new TableWriter(configuration, "x")) {
            for (int i = 0; i < M; i++) {
                TableWriter.Row row = writer.newRow(timestamp += increment);
                row.putStr(0, rnd.nextChars(20));
                row.putSym(1, symbols[rnd.nextPositiveInt() % N]);
                row.putInt(2, rnd.nextInt());
                row.putSym(3, symbols[rnd.nextPositiveInt() % N]);
                row.append();
            }
            writer.commit();
        }
        try (Engine engine = new Engine(configuration)) {
            RecordCursorFactory factory = new TableReaderRecordCursorFactory(engine, "x");
            long count = 0;
            RecordCursor cursor = factory.getCursor();
            try {
                rnd.reset();
                while (cursor.hasNext()) {
                    Record record = cursor.next();
                    TestUtils.assertEquals(rnd.nextChars(20), record.getFlyweightStr(0));
                    TestUtils.assertEquals(symbols[rnd.nextPositiveInt() % N], record.getSym(1));
                    Assert.assertEquals(rnd.nextInt(), record.getInt(2));
                    TestUtils.assertEquals(symbols[rnd.nextPositiveInt() % N], record.getSym(3));
                    count++;
                }
            } finally {
                cursor.releaseCursor();
            }
            Assert.assertEquals(0, engine.getBusyReaderCount());
            Assert.assertEquals(M, count);
        }
    });
}

Example 23 with RecordCursor

use of com.questdb.common.RecordCursor in project questdb by bluestreak01.

the class ReaderPoolTest method testConcurrentRead.

@Test
public void testConcurrentRead() throws Exception {
    final int readerCount = 5;
    int threadCount = 2;
    final int iterations = 1000000;
    Rnd dataRnd = new Rnd();
    final String[] names = new String[readerCount];
    final String[] expectedRows = new String[readerCount];
    final CharSequenceObjHashMap<String> expectedRowMap = new CharSequenceObjHashMap<>();
    for (int i = 0; i < readerCount; i++) {
        names[i] = "x" + i;
        try (TableModel model = new TableModel(configuration, names[i], PartitionBy.NONE).col("ts", ColumnType.DATE)) {
            CairoTestUtils.create(model);
        }
        try (TableWriter w = new TableWriter(configuration, names[i])) {
            for (int k = 0; k < 10; k++) {
                TableWriter.Row r = w.newRow(0);
                r.putDate(0, dataRnd.nextLong());
                r.append();
            }
            w.commit();
        }
        sink.clear();
        try (TableReader r = new TableReader(configuration, names[i])) {
            printer.print(r.getCursor(), true, r.getMetadata());
        }
        expectedRows[i] = sink.toString();
        expectedRowMap.put(names[i], expectedRows[i]);
    }
    assertWithPool((ReaderPool pool) -> {
        final CyclicBarrier barrier = new CyclicBarrier(threadCount);
        final CountDownLatch halt = new CountDownLatch(threadCount);
        final AtomicInteger errors = new AtomicInteger();
        for (int k = 0; k < threadCount; k++) {
            new Thread(new Runnable() {

                final ObjHashSet<TableReader> readers = new ObjHashSet<>();

                final StringSink sink = new StringSink();

                final RecordSourcePrinter printer = new RecordSourcePrinter(sink);

                @Override
                public void run() {
                    Rnd rnd = new Rnd();
                    try {
                        barrier.await();
                        String name;
                        // 3. it will close of of readers if has opened
                        for (int i = 0; i < iterations; i++) {
                            if (readers.size() == 0 || (readers.size() < 40 && rnd.nextPositiveInt() % 4 == 0)) {
                                name = names[rnd.nextPositiveInt() % readerCount];
                                try {
                                    Assert.assertTrue(readers.add(pool.get(name)));
                                } catch (EntryUnavailableException ignore) {
                                }
                            }
                            Thread.yield();
                            if (readers.size() == 0) {
                                continue;
                            }
                            int index = rnd.nextPositiveInt() % readers.size();
                            TableReader reader = readers.get(index);
                            Assert.assertTrue(reader.isOpen());
                            // read rows
                            RecordCursor cursor = reader.getCursor();
                            sink.clear();
                            printer.print(cursor, true, reader.getMetadata());
                            TestUtils.assertEquals(expectedRowMap.get(reader.getTableName()), sink);
                            Thread.yield();
                            if (readers.size() > 0 && rnd.nextPositiveInt() % 4 == 0) {
                                TableReader r2 = readers.get(rnd.nextPositiveInt() % readers.size());
                                Assert.assertTrue(r2.isOpen());
                                r2.close();
                                Assert.assertTrue(readers.remove(r2));
                            }
                            Thread.yield();
                        }
                    } catch (Exception e) {
                        errors.incrementAndGet();
                        e.printStackTrace();
                    } finally {
                        for (int i = 0; i < readers.size(); i++) {
                            readers.get(i).close();
                        }
                        halt.countDown();
                    }
                }
            }).start();
        }
        halt.await();
        Assert.assertEquals(0, halt.getCount());
        Assert.assertEquals(0, errors.get());
    });
}

Example 24 with RecordCursor

use of com.questdb.common.RecordCursor in project questdb by bluestreak01.

the class AbstractTest method assertSymbol.

public void assertSymbol(String query) throws ParserException {
    try (RecordSource src = compiler.compile(getFactory(), query)) {
        RecordCursor cursor = src.prepareCursor(getFactory());
        try {
            SymbolTable tab = cursor.getStorageFacade().getSymbolTable(0);
            while (cursor.hasNext()) {
                Record r = cursor.next();
                TestUtils.assertEquals(r.getSym(0), tab.value(r.getInt(0)));
            }
        } finally {
            cursor.releaseCursor();
        }
    }
}

Example 25 with RecordCursor

use of com.questdb.common.RecordCursor in project questdb by bluestreak01.

the class CachedAnalyticRecordSource method prepareCursor.

@Override
public RecordCursor prepareCursor(ReaderFactory factory, CancellationHandler cancellationHandler) {
    recordList.clear();
    for (int i = 0; i < orderGroupCount; i++) {
        RedBlackTree tree = orderedSources.getQuick(i);
        if (tree != null) {
            tree.clear();
        }
    }
    for (int i = 0, n = functions.size(); i < n; i++) {
        functions.getQuick(i).reset();
    }
    final RecordCursor cursor = recordSource.prepareCursor(factory, cancellationHandler);
    try {
        this.storageFacade.prepare(cursor.getStorageFacade());
        // step #1: store source cursor in record list
        // - add record list' row ids to all trees, which will put these row ids in necessary order
        // for this we will be using out comparator, which helps tree compare long values
        // based on record these values are addressing
        long rowid = -1;
        while (cursor.hasNext()) {
            cancellationHandler.check();
            Record record = cursor.next();
            rowid = recordList.append(record, rowid);
            if (orderGroupCount > 0) {
                for (int i = 0; i < orderGroupCount; i++) {
                    RedBlackTree tree = orderedSources.getQuick(i);
                    if (tree != null) {
                        tree.add(rowid);
                    }
                }
            }
        }
        for (int i = 0; i < orderGroupCount; i++) {
            RedBlackTree tree = orderedSources.getQuick(i);
            ObjList<AnalyticFunction> functions = functionGroups.getQuick(i);
            if (tree != null) {
                // step #2: populate all analytic functions with records in order of respective tree
                RedBlackTree.LongIterator iterator = tree.iterator();
                while (iterator.hasNext()) {
                    cancellationHandler.check();
                    Record record = recordList.recordAt(iterator.next());
                    for (int j = 0, n = functions.size(); j < n; j++) {
                        functions.getQuick(j).add(record);
                    }
                }
            } else {
                // step #2: alternatively run record list through two-pass functions
                for (int j = 0, n = functions.size(); j < n; j++) {
                    AnalyticFunction f = functions.getQuick(j);
                    if (f.getType() != AnalyticFunction.STREAM) {
                        recordList.toTop();
                        while (recordList.hasNext()) {
                            f.add(recordList.next());
                        }
                    }
                }
            }
        }
    } finally {
        cursor.releaseCursor();
    }
    recordList.toTop();
    setCursorAndPrepareFunctions();
    return this;
}