Examples with Row io.questdb.cairo.TableWriter.Row used on opensource projects

Example 1 with Row

use of io.questdb.cairo.TableWriter.Row in project questdb by bluestreak01.

the class O3CommitLagTest method insertUncommitted.

private void insertUncommitted(SqlCompiler compiler, SqlExecutionContext sqlExecutionContext, String sql, TableWriter writer) throws SqlException {
    minTimestamp = Long.MAX_VALUE;
    maxTimestamp = Long.MIN_VALUE;
    try (RecordCursorFactory factory = compiler.compile(sql, sqlExecutionContext).getRecordCursorFactory()) {
        RecordMetadata metadata = factory.getMetadata();
        int timestampIndex = writer.getMetadata().getTimestampIndex();
        EntityColumnFilter toColumnFilter = new EntityColumnFilter();
        toColumnFilter.of(metadata.getColumnCount());
        if (null == copier) {
            copier = SqlCompiler.assembleRecordToRowCopier(new BytecodeAssembler(), metadata, writer.getMetadata(), toColumnFilter);
        }
        try (RecordCursor cursor = factory.getCursor(sqlExecutionContext)) {
            final Record record = cursor.getRecord();
            while (cursor.hasNext()) {
                long timestamp = record.getTimestamp(timestampIndex);
                if (timestamp > maxTimestamp) {
                    maxTimestamp = timestamp;
                }
                if (timestamp < minTimestamp) {
                    minTimestamp = timestamp;
                }
                Row row = writer.newRow(timestamp);
                copier.copy(record, row);
                row.append();
            }
        }
    }
}

Example 2 with Row

use of io.questdb.cairo.TableWriter.Row in project questdb by bluestreak01.

the class O3CommitLagTest method testBigUncommittedMove1.

private void testBigUncommittedMove1(CairoEngine engine, SqlCompiler compiler, SqlExecutionContext sqlExecutionContext, TableModel tableModel) throws NumericException, SqlException {
    // Create empty tables of same structure
    TestUtils.createPopulateTable("o3", compiler, sqlExecutionContext, tableModel, 0, "2021-04-27", 0);
    TestUtils.createPopulateTable("ordered", compiler, sqlExecutionContext, tableModel, 0, "2021-04-27", 0);
    int longColIndex = -1;
    int flotColIndex = -1;
    int strColIndex = -1;
    for (int i = 0; i < tableModel.getColumnCount(); i++) {
        switch(ColumnType.tagOf(tableModel.getColumnType(i))) {
            case ColumnType.LONG:
                longColIndex = i;
                break;
            case ColumnType.FLOAT:
                flotColIndex = i;
                break;
            case ColumnType.STRING:
                strColIndex = i;
                break;
        }
    }
    long start = IntervalUtils.parseFloorPartialDate("2021-04-27T08:00:00");
    long[] testCounts = new long[] { 2 * 1024 * 1024, 16 * 8 * 1024 * 5, 2_000_000 };
    for (int c = 0; c < testCounts.length; c++) {
        long idCount = testCounts[c];
        // Create big commit with has big part before OOO starts
        // which exceed default MAMemoryImpl size in one or all columns
        int iterations = 2;
        String[] varCol = new String[] { "abc", "aldfjkasdlfkj", "as", "2021-04-27T12:00:00", "12345678901234578" };
        // Add 2 batches
        try (TableWriter o3 = engine.getWriter(AllowAllCairoSecurityContext.INSTANCE, "o3", "testing");
            TableWriter ordered = engine.getWriter(AllowAllCairoSecurityContext.INSTANCE, "ordered", "testing")) {
            for (int i = 0; i < iterations; i++) {
                long backwards = iterations - i - 1;
                final Rnd rnd = new Rnd();
                for (int id = 0; id < idCount; id++) {
                    long timestamp = start + backwards * idCount + id;
                    Row row = o3.newRow(timestamp);
                    if (longColIndex > -1) {
                        row.putLong(longColIndex, timestamp);
                    }
                    if (flotColIndex > -1) {
                        row.putFloat(flotColIndex, rnd.nextFloat());
                    }
                    if (strColIndex > -1) {
                        row.putStr(strColIndex, varCol[id % varCol.length]);
                    }
                    row.append();
                    timestamp = start + i * idCount + id;
                    row = ordered.newRow(timestamp);
                    if (longColIndex > -1) {
                        row.putLong(longColIndex, timestamp);
                    }
                    if (flotColIndex > -1) {
                        row.putFloat(flotColIndex, rnd.nextFloat());
                    }
                    if (strColIndex > -1) {
                        row.putStr(strColIndex, varCol[id % varCol.length]);
                    }
                    row.append();
                }
            }
            o3.commit();
            ordered.commit();
        }
        TestUtils.assertSqlCursors(compiler, sqlExecutionContext, "ordered", "o3", LOG);
        engine.releaseAllReaders();
        try (TableWriter o3 = engine.getWriter(AllowAllCairoSecurityContext.INSTANCE, "o3", "testing");
            TableWriter ordered = engine.getWriter(AllowAllCairoSecurityContext.INSTANCE, "ordered", "testing")) {
            o3.truncate();
            ordered.truncate();
        }
    }
}

Example 3 with Row

use of io.questdb.cairo.TableWriter.Row in project questdb by bluestreak01.

the class O3CommitLagTest method testRowCountWhenLagIsNextDay.

@Test
public void testRowCountWhenLagIsNextDay() throws Exception {
    executeWithPool(0, (CairoEngine engine, SqlCompiler compiler, SqlExecutionContext sqlExecutionContext) -> {
        String[] dates = new String[] { "2021-01-01T00:05:00.000000Z", "2021-01-01T00:01:00.000000Z", "2021-01-02T00:05:31.000000Z", "2021-01-01T00:01:30.000000Z", "2021-01-02T00:00:30.000000Z" };
        // Run same test taking 2-n lines from dates
        for (int length = 2; length <= dates.length; length++) {
            String tableName = "lll" + length;
            LOG.info().$("========= LENGTH ").$(length).$(" ================").$();
            compiler.compile("create table " + tableName + "( " + "ts timestamp" + ") timestamp(ts) partition by DAY " + " WITH maxUncommittedRows=1, commitLag=120s", sqlExecutionContext);
            try (TableWriter writer = engine.getWriter(AllowAllCairoSecurityContext.INSTANCE, tableName, "test")) {
                for (int i = 0; i < length; i++) {
                    long ts = IntervalUtils.parseFloorPartialDate(dates[i]);
                    Row r = writer.newRow(ts);
                    r.append();
                    Assert.assertEquals(i + 1, writer.size());
                    writer.checkMaxAndCommitLag(CommitMode.NOSYNC);
                    Assert.assertEquals(i + 1, writer.size());
                }
                writer.commit();
                sink.clear();
                TestUtils.printSql(compiler, sqlExecutionContext, "select count() from " + tableName, sink);
                TestUtils.assertEquals("count\n" + length + "\n", sink);
                Assert.assertEquals(length, writer.size());
                Assert.assertEquals(length > 2 ? 2 : 1, writer.getPartitionCount());
            }
        }
    });
}