Examples with MultiIntervalPartitionSource com.questdb.ql.interval.MultiIntervalPartitionSource used on opensource projects

Example 1 with MultiIntervalPartitionSource

use of com.questdb.ql.interval.MultiIntervalPartitionSource in project questdb by bluestreak01.

the class QueryCompiler method compileJournal0.

@SuppressWarnings("ConstantConditions")
private RecordSource compileJournal0(QueryModel model, Factory factory) throws ParserException {
    applyLimit(model);
    RecordMetadata metadata = model.getMetadata();
    JournalMetadata journalMetadata;
    if (metadata == null) {
        metadata = model.collectJournalMetadata(factory);
    }
    if (metadata instanceof JournalMetadata) {
        journalMetadata = (JournalMetadata) metadata;
    } else {
        throw QueryError.$(0, "Internal error: invalid metadata");
    }
    if (model.getAlias() != null) {
        journalMetadata.setAlias(model.getAlias().token);
    } else {
        journalMetadata.setAlias(QueryModel.stripMarker(model.getJournalName().token));
    }
    PartitionSource ps = new JournalPartitionSource(journalMetadata, true);
    RowSource rs = null;
    String latestByCol = null;
    RecordColumnMetadata latestByMetadata = null;
    ExprNode latestByNode = null;
    if (model.getLatestBy() != null) {
        latestByNode = model.getLatestBy();
        if (latestByNode.type != ExprNode.LITERAL) {
            throw QueryError.$(latestByNode.position, "Column name expected");
        }
        latestByCol = model.translateAlias(latestByNode.token).toString();
        int colIndex = journalMetadata.getColumnIndexQuiet(latestByCol);
        if (colIndex == -1) {
            throw QueryError.invalidColumn(latestByNode.position, latestByNode.token);
        }
        latestByMetadata = journalMetadata.getColumnQuick(colIndex);
        int type = latestByMetadata.getType();
        if (type != ColumnType.SYMBOL && type != ColumnType.STRING && type != ColumnType.INT && type != ColumnType.LONG) {
            throw QueryError.position(latestByNode.position).$("Expected symbol, string, int or long column, found: ").$(ColumnType.nameOf(type)).$();
        }
        if (!latestByMetadata.isIndexed()) {
            throw QueryError.position(latestByNode.position).$("Column is not indexed").$();
        }
    }
    ExprNode where = model.getWhereClause();
    if (where != null) {
        IntrinsicModel im = queryFilterAnalyser.extract(model, where, journalMetadata, latestByCol, getTimestampIndexQuiet(model.getTimestamp(), journalMetadata));
        VirtualColumn filter = im.filter != null ? virtualColumnBuilder.createVirtualColumn(model, im.filter, journalMetadata) : null;
        if (filter != null) {
            if (filter.getType() != ColumnType.BOOLEAN) {
                throw QueryError.$(im.filter.position, "Boolean expression expected");
            }
            if (filter.isConstant()) {
                if (filter.getBool(null)) {
                    // constant TRUE, no filtering needed
                    filter = null;
                } else {
                    im.intrinsicValue = IntrinsicValue.FALSE;
                }
            }
        }
        if (im.intrinsicValue == IntrinsicValue.FALSE) {
            ps = new NoOpJournalPartitionSource(journalMetadata);
        } else {
            if (im.intervals != null) {
                ps = new MultiIntervalPartitionSource(ps, im.intervals);
            }
            if (latestByCol == null) {
                if (im.keyColumn != null) {
                    switch(journalMetadata.getColumn(im.keyColumn).getType()) {
                        case ColumnType.SYMBOL:
                            rs = buildRowSourceForSym(im);
                            break;
                        case ColumnType.STRING:
                            rs = buildRowSourceForStr(im);
                            break;
                        case ColumnType.INT:
                            rs = buildRowSourceForInt(im);
                            break;
                        case ColumnType.LONG:
                            rs = buildRowSourceForLong(im);
                            break;
                        default:
                            break;
                    }
                }
                if (filter != null) {
                    rs = new FilteredRowSource(rs == null ? new AllRowSource() : rs, filter);
                }
            } else {
                if (im.keyColumn != null && im.keyValuesIsLambda) {
                    int lambdaColIndex;
                    RecordSource lambda = compileSourceInternal(factory, im.keyValues.get(0));
                    RecordMetadata m = lambda.getMetadata();
                    switch(m.getColumnCount()) {
                        case 0:
                            Misc.free(lambda);
                            throw QueryError.$(im.keyValuePositions.getQuick(0), "Query must select at least one column");
                        case 1:
                            lambdaColIndex = 0;
                            break;
                        default:
                            lambdaColIndex = m.getColumnIndexQuiet(latestByCol);
                            if (lambdaColIndex == -1) {
                                Misc.free(lambda);
                                throw QueryError.$(im.keyValuePositions.getQuick(0), "Ambiguous column names in lambda query. Specify select clause");
                            }
                            break;
                    }
                    int lambdaColType = m.getColumnQuick(lambdaColIndex).getType();
                    mutableSig.setParamCount(2).setName("").paramType(0, latestByMetadata.getType(), true).paramType(1, lambdaColType, false);
                    LatestByLambdaRowSourceFactory fact = LAMBDA_ROW_SOURCE_FACTORIES.get(mutableSig);
                    if (fact != null) {
                        rs = fact.newInstance(latestByCol, lambda, lambdaColIndex, filter);
                    } else {
                        Misc.free(lambda);
                        throw QueryError.$(im.keyValuePositions.getQuick(0), "Mismatched types");
                    }
                } else {
                    switch(latestByMetadata.getType()) {
                        case ColumnType.SYMBOL:
                            if (im.keyColumn != null) {
                                rs = new KvIndexSymListHeadRowSource(latestByCol, new CharSequenceHashSet(im.keyValues), filter);
                            } else {
                                rs = new KvIndexSymAllHeadRowSource(latestByCol, filter);
                            }
                            break;
                        case ColumnType.STRING:
                            if (im.keyColumn != null) {
                                rs = new KvIndexStrListHeadRowSource(latestByCol, new CharSequenceHashSet(im.keyValues), filter);
                            } else {
                                Misc.free(rs);
                                throw QueryError.$(latestByNode.position, "Filter on string column expected");
                            }
                            break;
                        case ColumnType.LONG:
                            if (im.keyColumn != null) {
                                rs = new KvIndexLongListHeadRowSource(latestByCol, toLongHashSet(im), filter);
                            } else {
                                Misc.free(rs);
                                throw QueryError.$(latestByNode.position, "Filter on long column expected");
                            }
                            break;
                        case ColumnType.INT:
                            if (im.keyColumn != null) {
                                rs = new KvIndexIntListHeadRowSource(latestByCol, toIntHashSet(im), filter);
                            } else {
                                Misc.free(rs);
                                throw QueryError.$(latestByNode.position, "Filter on int column expected");
                            }
                            break;
                        default:
                            break;
                    }
                }
            }
        }
    } else if (latestByCol != null) {
        switch(latestByMetadata.getType()) {
            case ColumnType.SYMBOL:
                rs = new KvIndexSymAllHeadRowSource(latestByCol, null);
                break;
            default:
                Misc.free(rs);
                throw QueryError.$(latestByNode.position, "Only SYM columns can be used here without filter");
        }
    }
    // check for case of simple "select count() from tab"
    if (rs == null && model.getColumns().size() == 1) {
        QueryColumn qc = model.getColumns().getQuick(0);
        if ("count".equals(qc.getAst().token) && qc.getAst().paramCount == 0) {
            // remove order clause
            model.getOrderBy().clear();
            // remove columns so that there is no wrapping of result source
            model.getColumns().clear();
            return new CountRecordSource(qc.getAlias() == null ? "count" : qc.getAlias(), ps);
        }
    }
    RecordSource recordSource = new JournalRecordSource(ps, rs == null ? new AllRowSource() : rs);
    if (QueryModel.hasMarker(model.getJournalName().token)) {
        return new NoRowIdRecordSource().of(recordSource);
    }
    return recordSource;
}

Example 2 with MultiIntervalPartitionSource

use of com.questdb.ql.interval.MultiIntervalPartitionSource in project questdb by bluestreak01.

the class MultiIntervalPartitionSourceTest method testIntervalMerge.

@Test
public void testIntervalMerge() throws Exception {
    StringSink sink = new StringSink();
    try (JournalWriter<Quote> w = getFactory().writer(Quote.class)) {
        TestUtils.generateQuoteData(w, 600, DateFormatUtils.parseDateTime("2014-03-10T02:00:00.000Z"), Dates.MINUTE_MILLIS);
        w.commit();
        RecordSourcePrinter p = new RecordSourcePrinter(sink);
        LongList intervals = new LongList();
        intervals.add(DateFormatUtils.parseDateTime("2014-03-10T07:00:00.000Z"));
        intervals.add(DateFormatUtils.parseDateTime("2014-03-10T07:15:00.000Z"));
        p.print(new JournalRecordSource(new MultiIntervalPartitionSource(new JournalPartitionSource(w.getMetadata(), true), intervals), new AllRowSource()), getFactory());
    }
    final String expected = "2014-03-10T07:00:00.000Z\tGKN.L\t290.000000000000\t320.000000000000\t1070060020\t627764827\tFast trading\tLXE\n" + "2014-03-10T07:01:00.000Z\tLLOY.L\t0.001271238521\t0.000000010817\t855783502\t444545168\tFast trading\tLXE\n" + "2014-03-10T07:02:00.000Z\tRRS.L\t0.000010917804\t272.000000000000\t1212565949\t1829154977\tFast trading\tLXE\n" + "2014-03-10T07:03:00.000Z\tTLW.L\t245.300086975098\t363.160156250000\t1722093204\t448833342\tFast trading\tLXE\n" + "2014-03-10T07:04:00.000Z\tTLW.L\t0.025095539168\t0.000000122058\t1703832336\t180642477\tFast trading\tLXE\n" + "2014-03-10T07:05:00.000Z\tADM.L\t902.500000000000\t24.333267211914\t781438951\t502201118\tFast trading\tLXE\n" + "2014-03-10T07:06:00.000Z\tAGK.L\t144.695419311523\t0.000039814179\t639071723\t1848238665\tFast trading\tLXE\n" + "2014-03-10T07:07:00.000Z\tLLOY.L\t0.000035416079\t15.248794555664\t1987214795\t856360285\tFast trading\tLXE\n" + "2014-03-10T07:08:00.000Z\tAGK.L\t0.207015849650\t0.199165701866\t1090730005\t1076974002\tFast trading\tLXE\n" + "2014-03-10T07:09:00.000Z\tLLOY.L\t447.510742187500\t209.001678466797\t136979290\t653726755\tFast trading\tLXE\n" + "2014-03-10T07:10:00.000Z\tBT-A.L\t662.032958984375\t0.000000007138\t1140333902\t1156896957\tFast trading\tLXE\n" + "2014-03-10T07:11:00.000Z\tAGK.L\t512.000000000000\t33.973937988281\t1723438228\t349327821\tFast trading\tLXE\n" + "2014-03-10T07:12:00.000Z\tWTB.L\t384.000000000000\t0.000000832384\t2145991300\t1388483923\tFast trading\tLXE\n" + "2014-03-10T07:13:00.000Z\tTLW.L\t0.000000093063\t0.000071085584\t1186156647\t1143726003\tFast trading\tLXE\n" + "2014-03-10T07:14:00.000Z\tAGK.L\t0.006215140224\t0.000000004051\t2086874501\t1272052914\tFast trading\tLXE\n" + "2014-03-10T07:15:00.000Z\tBP.L\t642.189208984375\t148.932441711426\t1552494421\t348870719\tFast trading\tLXE\n";
    Assert.assertEquals(expected, sink.toString());
}