Examples with TableReader io.questdb.cairo.TableReader used on opensource projects

Example 6 with TableReader

use of io.questdb.cairo.TableReader in project questdb by bluestreak01.

the class AlterTableRenameColumnTest method testRenameTimestampColumnAndCheckOpenReaderWithCursor.

@Test
public void testRenameTimestampColumnAndCheckOpenReaderWithCursor() throws Exception {
    String expectedBefore = "i\tsym\tamt\ttimestamp\tb\tc\td\te\tf\tg\tik\tj\tk\tl\tm\tn\n" + "1\tmsft\t50.938\t2018-01-01T00:12:00.000000Z\tfalse\tXYZ\t0.4621835429127854\t0.5599\t31\t2015-06-22T18:58:53.562Z\tPEHN\t-4485747798769957016\t1970-01-01T00:00:00.000000Z\t19\t00000000 19 c4 95 94 36 53 49 b4 59 7e 3b 08 a1 1e\tYSBEOUOJSHRUEDRQ\n" + "2\tgoogl\t42.281\t2018-01-01T00:24:00.000000Z\tfalse\tABC\t0.4138164748227684\t0.5522\t493\t2015-04-09T11:42:28.332Z\tHYRX\t-8811278461560712840\t1970-01-01T00:16:40.000000Z\t29\t00000000 53 d0 fb 64 bb 1a d4 f0 2d 40 e2 4b b1 3e e3 f1\t\n" + "3\tgoogl\t17.371\t2018-01-01T00:36:00.000000Z\tfalse\tABC\t0.05384400312338511\t0.0975\t327\t2015-09-26T18:05:10.217Z\tHYRX\t-3214230645884399728\t1970-01-01T00:33:20.000000Z\t28\t00000000 8e e5 61 2f 64 0e 2c 7f d7 6f b8 c9 ae 28\tSUWDSWUGS\n" + "4\tibm\t44.805\t2018-01-01T00:48:00.000000Z\ttrue\tXYZ\t0.14830552335848957\t0.9442\t95\t2015-01-04T19:58:55.654Z\tPEHN\t-5024542231726589509\t1970-01-01T00:50:00.000000Z\t39\t00000000 49 1c f2 3c ed 39 ac a8 3b a6\tOJIPHZEPIHVL\n" + "5\tgoogl\t42.956\t2018-01-01T01:00:00.000000Z\ttrue\t\t0.22895725920713628\t0.9821\t696\t2015-03-18T09:57:14.898Z\tCPSW\t-8757007522346766135\t1970-01-01T01:06:40.000000Z\t23\t\t\n" + "6\tibm\t82.59700000000001\t2018-01-01T01:12:00.000000Z\ttrue\tCDE\t0.021189232728939578\tNaN\t369\t2015-07-21T10:33:47.953Z\tPEHN\t-9147563299122452591\t1970-01-01T01:23:20.000000Z\t48\t00000000 18 93 bd 0b 61 f5 5d d0 eb 67 44 a7 6a\t\n" + "7\tgoogl\t98.59100000000001\t2018-01-01T01:24:00.000000Z\ttrue\tCDE\t0.24642266252221556\t0.2672\t174\t2015-02-20T01:11:53.748Z\t\t2151565237758036093\t1970-01-01T01:40:00.000000Z\t31\t\tHZSQLDGLOGIFO\n" + "8\tgoogl\t57.086\t2018-01-01T01:36:00.000000Z\ttrue\tABC\t0.6707018622395736\t0.0759\t199\t2015-09-12T07:21:40.050Z\t\t-4058426794463997577\t1970-01-01T01:56:40.000000Z\t37\t00000000 ea 4e ea 8b f5 0f 2d b3 14 33\tFFLRBROMNXKUIZ\n" + "9\tgoogl\t81.44200000000001\t2018-01-01T01:48:00.000000Z\tfalse\t\t0.2677326840703891\t0.5425\t1001\t2015-11-14T07:05:22.934Z\tHYRX\t-8793423647053878901\t1970-01-01T02:13:20.000000Z\t33\t00000000 25 c2 20 ff 70 3a c7 8a b3 14 cd 47 0b 0c\tFMQNTOG\n" + "10\tmsft\t3.973\t2018-01-01T02:00:00.000000Z\tfalse\tXYZ\tNaN\tNaN\t828\t2015-06-18T18:07:42.406Z\tPEHN\t-7398902448022205322\t1970-01-01T02:30:00.000000Z\t50\t00000000 fb 2e 42 fa f5 6e 8f 80 e3 54 b8 07 b1 32 57 ff\n" + "00000010 9a ef 88 cb\tCNGTNLEGPUHH\n";
    String expectedAfter = "i\tsym\tamt\tts\tb\tc\td\te\tf\tg\tik\tj\tk\tl\tm\tn\n" + "1\tmsft\t50.938\t2018-01-01T00:12:00.000000Z\tfalse\tXYZ\t0.4621835429127854\t0.5599\t31\t2015-06-22T18:58:53.562Z\tPEHN\t-4485747798769957016\t1970-01-01T00:00:00.000000Z\t19\t00000000 19 c4 95 94 36 53 49 b4 59 7e 3b 08 a1 1e\tYSBEOUOJSHRUEDRQ\n" + "2\tgoogl\t42.281\t2018-01-01T00:24:00.000000Z\tfalse\tABC\t0.4138164748227684\t0.5522\t493\t2015-04-09T11:42:28.332Z\tHYRX\t-8811278461560712840\t1970-01-01T00:16:40.000000Z\t29\t00000000 53 d0 fb 64 bb 1a d4 f0 2d 40 e2 4b b1 3e e3 f1\t\n" + "3\tgoogl\t17.371\t2018-01-01T00:36:00.000000Z\tfalse\tABC\t0.05384400312338511\t0.0975\t327\t2015-09-26T18:05:10.217Z\tHYRX\t-3214230645884399728\t1970-01-01T00:33:20.000000Z\t28\t00000000 8e e5 61 2f 64 0e 2c 7f d7 6f b8 c9 ae 28\tSUWDSWUGS\n" + "4\tibm\t44.805\t2018-01-01T00:48:00.000000Z\ttrue\tXYZ\t0.14830552335848957\t0.9442\t95\t2015-01-04T19:58:55.654Z\tPEHN\t-5024542231726589509\t1970-01-01T00:50:00.000000Z\t39\t00000000 49 1c f2 3c ed 39 ac a8 3b a6\tOJIPHZEPIHVL\n" + "5\tgoogl\t42.956\t2018-01-01T01:00:00.000000Z\ttrue\t\t0.22895725920713628\t0.9821\t696\t2015-03-18T09:57:14.898Z\tCPSW\t-8757007522346766135\t1970-01-01T01:06:40.000000Z\t23\t\t\n" + "6\tibm\t82.59700000000001\t2018-01-01T01:12:00.000000Z\ttrue\tCDE\t0.021189232728939578\tNaN\t369\t2015-07-21T10:33:47.953Z\tPEHN\t-9147563299122452591\t1970-01-01T01:23:20.000000Z\t48\t00000000 18 93 bd 0b 61 f5 5d d0 eb 67 44 a7 6a\t\n" + "7\tgoogl\t98.59100000000001\t2018-01-01T01:24:00.000000Z\ttrue\tCDE\t0.24642266252221556\t0.2672\t174\t2015-02-20T01:11:53.748Z\t\t2151565237758036093\t1970-01-01T01:40:00.000000Z\t31\t\tHZSQLDGLOGIFO\n" + "8\tgoogl\t57.086\t2018-01-01T01:36:00.000000Z\ttrue\tABC\t0.6707018622395736\t0.0759\t199\t2015-09-12T07:21:40.050Z\t\t-4058426794463997577\t1970-01-01T01:56:40.000000Z\t37\t00000000 ea 4e ea 8b f5 0f 2d b3 14 33\tFFLRBROMNXKUIZ\n" + "9\tgoogl\t81.44200000000001\t2018-01-01T01:48:00.000000Z\tfalse\t\t0.2677326840703891\t0.5425\t1001\t2015-11-14T07:05:22.934Z\tHYRX\t-8793423647053878901\t1970-01-01T02:13:20.000000Z\t33\t00000000 25 c2 20 ff 70 3a c7 8a b3 14 cd 47 0b 0c\tFMQNTOG\n" + "10\tmsft\t3.973\t2018-01-01T02:00:00.000000Z\tfalse\tXYZ\tNaN\tNaN\t828\t2015-06-18T18:07:42.406Z\tPEHN\t-7398902448022205322\t1970-01-01T02:30:00.000000Z\t50\t00000000 fb 2e 42 fa f5 6e 8f 80 e3 54 b8 07 b1 32 57 ff\n" + "00000010 9a ef 88 cb\tCNGTNLEGPUHH\n";
    assertMemoryLeak(() -> {
        createX();
        try (TableReader reader = engine.getReader(sqlExecutionContext.getCairoSecurityContext(), "x")) {
            Assert.assertEquals("timestamp", reader.getMetadata().getColumnName(3));
            // check cursor before renaming column
            TestUtils.assertReader(expectedBefore, reader, sink);
            try (TableWriter writer = engine.getWriter(sqlExecutionContext.getCairoSecurityContext(), "x", "testing")) {
                writer.renameColumn("timestamp", "ts");
                Assert.assertEquals("ts", writer.getMetadata().getColumnName(3));
            }
            // reload reader
            Assert.assertTrue(reader.reload());
            // check cursor after reload
            TestUtils.assertReader(expectedAfter, reader, sink);
            assertReader(expectedAfter, "x");
            Assert.assertEquals("ts", reader.getMetadata().getColumnName(3));
        }
    });
}

Example 7 with TableReader

use of io.questdb.cairo.TableReader in project questdb by bluestreak01.

the class AlterTableRenameColumnTest method testRenameColumn.

@Test
public void testRenameColumn() throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        try {
            createX();
            Assert.assertEquals(ALTER, compiler.compile("alter table x rename column e to z", sqlExecutionContext).getType());
            String expected = "{\"columnCount\":16,\"columns\":[{\"index\":0,\"name\":\"i\",\"type\":\"INT\"},{\"index\":1,\"name\":\"sym\",\"type\":\"SYMBOL\"},{\"index\":2,\"name\":\"amt\",\"type\":\"DOUBLE\"},{\"index\":3,\"name\":\"timestamp\",\"type\":\"TIMESTAMP\"},{\"index\":4,\"name\":\"b\",\"type\":\"BOOLEAN\"},{\"index\":5,\"name\":\"c\",\"type\":\"STRING\"},{\"index\":6,\"name\":\"d\",\"type\":\"DOUBLE\"},{\"index\":7,\"name\":\"z\",\"type\":\"FLOAT\"},{\"index\":8,\"name\":\"f\",\"type\":\"SHORT\"},{\"index\":9,\"name\":\"g\",\"type\":\"DATE\"},{\"index\":10,\"name\":\"ik\",\"type\":\"SYMBOL\"},{\"index\":11,\"name\":\"j\",\"type\":\"LONG\"},{\"index\":12,\"name\":\"k\",\"type\":\"TIMESTAMP\"},{\"index\":13,\"name\":\"l\",\"type\":\"BYTE\"},{\"index\":14,\"name\":\"m\",\"type\":\"BINARY\"},{\"index\":15,\"name\":\"n\",\"type\":\"STRING\"}],\"timestampIndex\":3}";
            try (TableReader reader = engine.getReader(AllowAllCairoSecurityContext.INSTANCE, "x", TableUtils.ANY_TABLE_VERSION, TableUtils.ANY_TABLE_VERSION)) {
                sink.clear();
                reader.getMetadata().toJson(sink);
                TestUtils.assertEquals(expected, sink);
            }
            Assert.assertEquals(0, engine.getBusyWriterCount());
            Assert.assertEquals(0, engine.getBusyReaderCount());
        } finally {
            engine.clear();
        }
    });
}

Example 8 with TableReader

use of io.questdb.cairo.TableReader in project questdb by bluestreak01.

the class AlterTableAlterSymbolColumnCacheFlagTest method testAlterSymbolCacheFlagToFalseAndCheckOpenReaderWithCursor.

@Test
public void testAlterSymbolCacheFlagToFalseAndCheckOpenReaderWithCursor() throws Exception {
    String expectedOrdered = "sym\n" + "googl\n" + "googl\n" + "googl\n" + "googl\n" + "googl\n" + "googl\n" + "ibm\n" + "ibm\n" + "msft\n" + "msft\n";
    String expectedChronological = "sym\n" + "msft\n" + "googl\n" + "googl\n" + "ibm\n" + "googl\n" + "ibm\n" + "googl\n" + "googl\n" + "googl\n" + "msft\n";
    final RecordCursorPrinter printer = new SingleColumnRecordCursorPrinter(1);
    assertMemoryLeak(() -> {
        assertMemoryLeak(this::createX);
        assertQueryPlain(expectedOrdered, "select sym from x order by sym");
        try (TableReader reader = engine.getReader(sqlExecutionContext.getCairoSecurityContext(), "x")) {
            // check cursor before altering symbol column
            sink.clear();
            printer.print(reader.getCursor(), reader.getMetadata(), true, sink);
            Assert.assertEquals(expectedChronological, sink.toString());
            try (TableWriter writer = engine.getWriter(sqlExecutionContext.getCairoSecurityContext(), "x", "testing")) {
                writer.changeCacheFlag(1, false);
            }
            // reload reader
            Assert.assertTrue(reader.reload());
            // check cursor after reload
            sink.clear();
            printer.print(reader.getCursor(), reader.getMetadata(), true, sink);
            Assert.assertEquals(expectedChronological, sink.toString());
            try (TableReader reader2 = engine.getReader(sqlExecutionContext.getCairoSecurityContext(), "x")) {
                sink.clear();
                printer.print(reader2.getCursor(), reader2.getMetadata(), true, sink);
                Assert.assertEquals(expectedChronological, sink.toString());
            }
        }
    });
    assertQueryPlain(expectedOrdered, "select sym from x order by 1 asc");
}

Example 9 with TableReader

use of io.questdb.cairo.TableReader in project questdb by bluestreak01.

the class TelemetryTest method assertColumn.

protected void assertColumn(CharSequence expected, int index) {
    try (TableReader reader = new TableReader(configuration, "telemetry")) {
        sink.clear();
        printer.printFullColumn(reader.getCursor(), reader.getMetadata(), index, false, sink);
        TestUtils.assertEquals(expected, sink);
        reader.getCursor().toTop();
        sink.clear();
        printer.printFullColumn(reader.getCursor(), reader.getMetadata(), index, false, sink);
        TestUtils.assertEquals(expected, sink);
    }
}

Example 10 with TableReader

use of io.questdb.cairo.TableReader in project questdb by bluestreak01.

the class LatestByAllIndexedRecordCursor method buildTreeMap.

@Override
protected void buildTreeMap(SqlExecutionContext executionContext) throws SqlException {
    final MessageBus bus = executionContext.getMessageBus();
    final RingQueue<LatestByTask> queue = bus.getLatestByQueue();
    final Sequence pubSeq = bus.getLatestByPubSeq();
    final Sequence subSeq = bus.getLatestBySubSeq();
    int keyCount = getSymbolTable(columnIndex).size() + 1;
    rows.extend(keyCount);
    GeoHashNative.iota(rows.getAddress(), rows.getCapacity(), 0);
    final int workerCount = executionContext.getWorkerCount();
    final long chunkSize = (keyCount + workerCount - 1) / workerCount;
    final int taskCount = (int) ((keyCount + chunkSize - 1) / chunkSize);
    final long argumentsAddress = LatestByArguments.allocateMemoryArray(taskCount);
    for (long i = 0; i < taskCount; ++i) {
        final long klo = i * chunkSize;
        final long khi = Long.min(klo + chunkSize, keyCount);
        final long argsAddress = argumentsAddress + i * LatestByArguments.MEMORY_SIZE;
        LatestByArguments.setRowsAddress(argsAddress, rows.getAddress());
        LatestByArguments.setRowsCapacity(argsAddress, rows.getCapacity());
        LatestByArguments.setKeyLo(argsAddress, klo);
        LatestByArguments.setKeyHi(argsAddress, khi);
        LatestByArguments.setRowsSize(argsAddress, 0);
    }
    int hashColumnIndex = -1;
    int hashColumnType = ColumnType.UNDEFINED;
    long prefixesAddress = 0;
    long prefixesCount = 0;
    if (this.prefixes.size() > 2) {
        hashColumnIndex = (int) prefixes.get(0);
        hashColumnType = (int) prefixes.get(1);
        prefixesAddress = prefixes.getAddress() + 2 * Long.BYTES;
        prefixesCount = prefixes.size() - 2;
    }
    DataFrame frame;
    // frame metadata is based on TableReader, which is "full" metadata
    // this cursor works with subset of columns, which warrants column index remap
    int frameColumnIndex = columnIndexes.getQuick(columnIndex);
    final TableReader reader = this.dataFrameCursor.getTableReader();
    long foundRowCount = 0;
    while ((frame = this.dataFrameCursor.next()) != null && foundRowCount < keyCount) {
        doneLatch.reset();
        final BitmapIndexReader indexReader = frame.getBitmapIndexReader(frameColumnIndex, BitmapIndexReader.DIR_BACKWARD);
        final long rowLo = frame.getRowLo();
        final long rowHi = frame.getRowHi() - 1;
        final long keyBaseAddress = indexReader.getKeyBaseAddress();
        final long keysMemorySize = indexReader.getKeyMemorySize();
        final long valueBaseAddress = indexReader.getValueBaseAddress();
        final long valuesMemorySize = indexReader.getValueMemorySize();
        final int valueBlockCapacity = indexReader.getValueBlockCapacity();
        final long unIndexedNullCount = indexReader.getUnIndexedNullCount();
        final int partitionIndex = frame.getPartitionIndex();
        long hashColumnAddress = 0;
        // hashColumnIndex can be -1 for latest by part only (no prefixes to match)
        if (hashColumnIndex > -1) {
            final int columnBase = reader.getColumnBase(partitionIndex);
            final int primaryColumnIndex = TableReader.getPrimaryColumnIndex(columnBase, hashColumnIndex);
            final MemoryR column = reader.getColumn(primaryColumnIndex);
            hashColumnAddress = column.getPageAddress(0);
        }
        // -1 must be dead case here
        final int hashesColumnSize = ColumnType.isGeoHash(hashColumnType) ? getPow2SizeOfGeoHashType(hashColumnType) : -1;
        int queuedCount = 0;
        for (long i = 0; i < taskCount; ++i) {
            final long argsAddress = argumentsAddress + i * LatestByArguments.MEMORY_SIZE;
            final long found = LatestByArguments.getRowsSize(argsAddress);
            final long keyHi = LatestByArguments.getKeyHi(argsAddress);
            final long keyLo = LatestByArguments.getKeyLo(argsAddress);
            // Skip range if all keys found
            if (found >= keyHi - keyLo) {
                continue;
            }
            // Update hash column address with current frame value
            LatestByArguments.setHashesAddress(argsAddress, hashColumnAddress);
            final long seq = pubSeq.next();
            if (seq < 0) {
                GeoHashNative.latestByAndFilterPrefix(keyBaseAddress, keysMemorySize, valueBaseAddress, valuesMemorySize, argsAddress, unIndexedNullCount, rowHi, rowLo, partitionIndex, valueBlockCapacity, hashColumnAddress, hashesColumnSize, prefixesAddress, prefixesCount);
            } else {
                queue.get(seq).of(keyBaseAddress, keysMemorySize, valueBaseAddress, valuesMemorySize, argsAddress, unIndexedNullCount, rowHi, rowLo, partitionIndex, valueBlockCapacity, hashColumnAddress, hashesColumnSize, prefixesAddress, prefixesCount, doneLatch);
                pubSeq.done(seq);
                queuedCount++;
            }
        }
        // this should fix deadlock with 1 worker configuration
        while (doneLatch.getCount() > -queuedCount) {
            long seq = subSeq.next();
            if (seq > -1) {
                queue.get(seq).run();
                subSeq.done(seq);
            }
        }
        doneLatch.await(queuedCount);
        // Reset found counter
        foundRowCount = 0;
        for (int i = 0; i < taskCount; i++) {
            final long address = argumentsAddress + i * LatestByArguments.MEMORY_SIZE;
            foundRowCount += LatestByArguments.getRowsSize(address);
        }
    }
    final long rowCount = GeoHashNative.slideFoundBlocks(argumentsAddress, taskCount);
    LatestByArguments.releaseMemoryArray(argumentsAddress, taskCount);
    aLimit = rowCount;
    aIndex = indexShift;
    postProcessRows();
}