Examples with RowCursor io.questdb.cairo.sql.RowCursor used on opensource projects

Example 11 with RowCursor

use of io.questdb.cairo.sql.RowCursor in project questdb by bluestreak01.

the class BitmapIndexTest method testBackwardReaderDoesNotUnmapPages.

@Test
public void testBackwardReaderDoesNotUnmapPages() throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        Rnd rnd = new Rnd();
        create(configuration, path.trimTo(plen), "x", 1024);
        class CountingFacade extends FilesFacadeImpl {

            private int count = 0;

            @Override
            public long getMapPageSize() {
                return 1024 * 1024;
            }

            @Override
            public void munmap(long address, long size, int memoryTag) {
                super.munmap(address, size, memoryTag);
                count++;
            }
        }
        final CountingFacade facade = new CountingFacade();
        CairoConfiguration configuration = new DefaultCairoConfiguration(root) {

            @Override
            public FilesFacade getFilesFacade() {
                return facade;
            }
        };
        try (BitmapIndexWriter writer = new BitmapIndexWriter(configuration, path.trimTo(plen), "x")) {
            try (BitmapIndexBwdReader reader = new BitmapIndexBwdReader(configuration, path.trimTo(plen), "x", 0)) {
                for (int i = 0; i < 100000; i++) {
                    int key = rnd.nextPositiveInt() % 1024;
                    long value = rnd.nextPositiveLong();
                    writer.add(key, value);
                    RowCursor cursor = reader.getCursor(true, key, 0, Long.MAX_VALUE);
                    boolean found = false;
                    while (cursor.hasNext()) {
                        if (value == cursor.next()) {
                            found = true;
                            break;
                        }
                    }
                    Assert.assertTrue(found);
                }
                Assert.assertEquals(0, facade.count);
            }
        }
    });
}

Example 12 with RowCursor

use of io.questdb.cairo.sql.RowCursor in project questdb by bluestreak01.

the class BitmapIndexTest method testTruncate.

@Test
public void testTruncate() {
    create(configuration, path.trimTo(plen), "x", 1024);
    int count = 2_500;
    // add multiple keys, 1,2,3,4,5,6,7... etc
    try (BitmapIndexWriter writer = new BitmapIndexWriter(configuration, path, "x")) {
        for (int i = 0; i < count; i++) {
            writer.add(i, 1000);
        }
        // check that these keys exist in the index
        Assert.assertEquals(count, writer.getKeyCount());
        for (int i = 0; i < count; i++) {
            RowCursor cursor = writer.getCursor(i);
            Assert.assertTrue(cursor.hasNext());
            Assert.assertEquals(1000, cursor.next());
            Assert.assertFalse(cursor.hasNext());
        }
        writer.truncate();
        Assert.assertEquals(0, writer.getKeyCount());
        // now add middle key first
        writer.add(900, 8000);
        Assert.assertEquals(901, writer.getKeyCount());
        try (BitmapIndexReader reader = new BitmapIndexBwdReader(configuration, path, "x", 0)) {
            Assert.assertEquals(901, reader.getKeyCount());
            RowCursor cursor = reader.getCursor(true, 900, 0, 1_000_000);
            Assert.assertTrue(cursor.hasNext());
            Assert.assertEquals(8000, cursor.next());
            Assert.assertFalse(cursor.hasNext());
            // assert that key below 900 do no have values
            for (int i = 0; i < 900; i++) {
                Assert.assertFalse(reader.getCursor(true, i, 0, 1_000_000).hasNext());
            }
        }
    }
}

Example 13 with RowCursor

use of io.questdb.cairo.sql.RowCursor in project questdb by bluestreak01.

the class BitmapIndexTest method testConcurrentBackwardRW.

private void testConcurrentBackwardRW(int N, int maxKeys) throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        Rnd rnd = new Rnd();
        IntList keys = new IntList();
        IntObjHashMap<LongList> lists = new IntObjHashMap<>();
        // populate model for both reader and writer
        for (int i = 0; i < N; i++) {
            int key = rnd.nextPositiveInt() % maxKeys;
            LongList list = lists.get(key);
            if (list == null) {
                lists.put(key, list = new LongList());
                keys.add(key);
            }
            list.add(i);
        }
        final int threadCount = 3;
        CountDownLatch stopLatch = new CountDownLatch(threadCount);
        CyclicBarrier startBarrier = new CyclicBarrier(threadCount);
        AtomicInteger errors = new AtomicInteger();
        // create empty index
        create(configuration, path.trimTo(plen), "x", 1024);
        new Thread(() -> {
            try {
                startBarrier.await();
                try (Path path = new Path().of(configuration.getRoot())) {
                    try (BitmapIndexWriter writer = new BitmapIndexWriter(configuration, path, "x")) {
                        int pass = 0;
                        while (true) {
                            boolean added = false;
                            for (int i = 0, n = keys.size(); i < n; i++) {
                                int key = keys.getQuick(i);
                                LongList values = lists.get(key);
                                if (pass < values.size()) {
                                    writer.add(key, values.getQuick(pass));
                                    added = true;
                                }
                            }
                            pass++;
                            if (!added) {
                                break;
                            }
                        }
                    }
                }
            } catch (Throwable e) {
                e.printStackTrace();
                errors.incrementAndGet();
            } finally {
                stopLatch.countDown();
            }
        }).start();
        class MyReader implements Runnable {

            @Override
            public void run() {
                try {
                    startBarrier.await();
                    try (Path path = new Path().of(configuration.getRoot())) {
                        try (BitmapIndexBwdReader reader1 = new BitmapIndexBwdReader(configuration, path, "x", 0)) {
                            LongList tmp = new LongList();
                            while (true) {
                                boolean keepGoing = false;
                                for (int i = keys.size() - 1; i > -1; i--) {
                                    int key = keys.getQuick(i);
                                    LongList values = lists.get(key);
                                    RowCursor cursor = reader1.getCursor(true, key, 0, Long.MAX_VALUE);
                                    tmp.clear();
                                    while (cursor.hasNext()) {
                                        tmp.add(cursor.next());
                                    }
                                    int sz = tmp.size();
                                    for (int k = 0; k < sz; k++) {
                                        Assert.assertEquals(values.getQuick(sz - k - 1), tmp.getQuick(k));
                                    }
                                    if (sz < values.size()) {
                                        keepGoing = true;
                                    }
                                }
                                if (!keepGoing) {
                                    break;
                                }
                            }
                        }
                    }
                } catch (Throwable e) {
                    errors.incrementAndGet();
                    e.printStackTrace();
                } finally {
                    stopLatch.countDown();
                }
            }
        }
        new Thread(new MyReader()).start();
        new Thread(new MyReader()).start();
        Assert.assertTrue(stopLatch.await(20000, TimeUnit.SECONDS));
        Assert.assertEquals(0, errors.get());
    });
}

Example 14 with RowCursor

use of io.questdb.cairo.sql.RowCursor in project questdb by bluestreak01.

the class BitmapIndexTest method testAdd1MValues.

@Test
public void testAdd1MValues() throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        Rnd rnd = new Rnd();
        int maxKeys = 1024;
        int N = 1000000;
        IntList keys = new IntList();
        IntObjHashMap<LongList> lists = new IntObjHashMap<>();
        // we need to have a conventional structure, which will unfortunately be memory-inefficient, to
        // assert that index is populated correctly
        create(configuration, path.trimTo(plen), "x", 128);
        try (BitmapIndexWriter writer = new BitmapIndexWriter(configuration, path, "x")) {
            for (int i = 0; i < N; i++) {
                int key = i % maxKeys;
                long value = rnd.nextLong();
                writer.add(key, value);
                LongList list = lists.get(key);
                if (list == null) {
                    lists.put(key, list = new LongList());
                    keys.add(key);
                }
                list.add(value);
            }
        }
        // read values and compare to the structure index is expected to be holding
        try (BitmapIndexBwdReader reader = new BitmapIndexBwdReader(configuration, path.trimTo(plen), "x", 0)) {
            for (int i = 0, n = keys.size(); i < n; i++) {
                LongList list = lists.get(keys.getQuick(i));
                Assert.assertNotNull(list);
                RowCursor cursor = reader.getCursor(true, keys.getQuick(i), Long.MIN_VALUE, Long.MAX_VALUE);
                int z = list.size();
                while (cursor.hasNext()) {
                    Assert.assertTrue(z > -1);
                    Assert.assertEquals(list.getQuick(z - 1), cursor.next());
                    z--;
                }
                // makes sure entire list is processed
                Assert.assertEquals(0, z);
            }
        }
        // read values and compare to the structure index is expected to be holding
        try (BitmapIndexFwdReader reader = new BitmapIndexFwdReader(configuration, path.trimTo(plen), "x", 0)) {
            for (int i = 0, n = keys.size(); i < n; i++) {
                LongList list = lists.get(keys.getQuick(i));
                Assert.assertNotNull(list);
                RowCursor cursor = reader.getCursor(true, keys.getQuick(i), Long.MIN_VALUE, Long.MAX_VALUE);
                int z = 0;
                int sz = list.size();
                while (cursor.hasNext()) {
                    Assert.assertTrue(z < sz);
                    Assert.assertEquals(list.getQuick(z), cursor.next());
                    z++;
                }
                // makes sure entire list is processed
                Assert.assertEquals(sz, z);
            }
        }
    });
}

Example 15 with RowCursor

use of io.questdb.cairo.sql.RowCursor in project questdb by bluestreak01.

the class BitmapIndexTest method testBackwardReaderKeyUpdateFail.

@Test
public void testBackwardReaderKeyUpdateFail() {
    create(configuration, path.trimTo(plen), "x", 1024);
    try (BitmapIndexWriter writer = new BitmapIndexWriter(configuration, path, "x")) {
        writer.add(0, 1000);
    }
    try (BitmapIndexBwdReader reader = new BitmapIndexBwdReader(configuration, path.trimTo(plen), "x", 0)) {
        // should have single value in cursor
        RowCursor cursor = reader.getCursor(true, 0, 0, Long.MAX_VALUE);
        Assert.assertTrue(cursor.hasNext());
        Assert.assertEquals(1000, cursor.next());
        Assert.assertFalse(cursor.hasNext());
        try (Path path = new Path();
            MemoryCMARW mem = Vm.getSmallCMARWInstance(configuration.getFilesFacade(), path.of(root).concat("x").put(".k").$(), MemoryTag.MMAP_DEFAULT)) {
            // change sequence but not sequence check
            long seq = mem.getLong(BitmapIndexUtils.KEY_RESERVED_OFFSET_SEQUENCE);
            mem.putLong(BitmapIndexUtils.KEY_RESERVED_OFFSET_SEQUENCE, 22);
            try {
                reader.getCursor(true, 10, 0, Long.MAX_VALUE);
            } catch (CairoException e) {
                Assert.assertTrue(Chars.contains(e.getMessage(), "could not consistently"));
            }
            try {
                reader.getCursor(true, 0, 0, Long.MAX_VALUE);
            } catch (CairoException e) {
                Assert.assertTrue(Chars.contains(e.getMessage(), "could not consistently"));
            }
            mem.putLong(BitmapIndexUtils.KEY_RESERVED_OFFSET_SEQUENCE, seq);
            // test that index recovers
            cursor = reader.getCursor(true, 0, 0, Long.MAX_VALUE);
            Assert.assertTrue(cursor.hasNext());
            Assert.assertEquals(1000, cursor.next());
            Assert.assertFalse(cursor.hasNext());
        }
    }
}