Examples with VirtualColumn com.questdb.ql.ops.VirtualColumn used on opensource projects

Example 1 with VirtualColumn

use of com.questdb.ql.ops.VirtualColumn in project questdb by bluestreak01.

the class QueryCompiler method analyseAndCompileOrderBy.

private RecordSource analyseAndCompileOrderBy(QueryModel model, RecordSource recordSource) throws ParserException {
    literalCollectorANames.clear();
    literalCollectorAIndexes.clear();
    literalCollector.withParent(model);
    literalCollector.resetNullCount();
    // create hash map keyed by column name for fast lookup access
    CharSequenceObjHashMap<QueryColumn> columnHashMap = new CharSequenceObjHashMap<>();
    ObjList<QueryColumn> columns = model.getColumns();
    for (int i = 0, n = columns.size(); i < n; i++) {
        final QueryColumn c = columns.getQuick(i);
        columnHashMap.put(c.getAlias() == null ? c.getName() : c.getAlias(), c);
    }
    ObjList<ExprNode> orderBy = model.getOrderBy();
    // determine where order by columns belong to
    innerVirtualColumn.clear();
    for (int i = 0, n = orderBy.size(); i < n; i++) {
        QueryColumn col = columnHashMap.get(orderBy.getQuick(i).token);
        if (col != null) {
            traversalAlgo.traverse(col.getAst(), literalCollector.lhs());
            if (col.getAst().type != ExprNode.LITERAL) {
                innerVirtualColumn.add(col);
            }
        } else {
            traversalAlgo.traverse(orderBy.getQuick(i), literalCollector.lhs());
        }
    }
    // we interested in columns that belong to journal 0 only
    // abort if we find references to anywhere else
    boolean canUseOrderByHere = true;
    for (int i = 0, n = literalCollectorAIndexes.size(); i < n; i++) {
        if (literalCollectorAIndexes.getQuick(i) != 0) {
            canUseOrderByHere = false;
            break;
        }
    }
    // if order by refers to an expression
    if (canUseOrderByHere) {
        RecordSource rs = recordSource;
        if (innerVirtualColumn.size() > 0) {
            ObjList<VirtualColumn> virtualColumns = new ObjList<>();
            for (int i = 0, n = innerVirtualColumn.size(); i < n; i++) {
                QueryColumn qc = innerVirtualColumn.getQuick(i);
                VirtualColumn vc = virtualColumnBuilder.createVirtualColumn(model, qc.getAst(), rs.getMetadata());
                vc.setName(qc.getAlias());
                virtualColumns.add(vc);
                // make query column a literal to prevent another virtual column being created
                // further down the pipeline
                ExprNode expr = qc.getAst();
                expr.type = ExprNode.LITERAL;
                expr.token = qc.getAlias();
                expr.args.clear();
                expr.lhs = null;
                expr.rhs = null;
                expr.position = qc.getAliasPosition();
                expr.paramCount = 0;
            }
            rs = new VirtualColumnRecordSource(rs, virtualColumns);
        }
        for (int i = 0, n = orderBy.size(); i < n; i++) {
            QueryColumn col = columnHashMap.get(orderBy.getQuick(i).token);
            if (col != null && col.getAst().type == ExprNode.LITERAL && col.getAlias() != null) {
                orderBy.getQuick(i).token = col.getAst().token;
            }
        }
        rs = order(rs, model);
        model.getOrderBy().clear();
        return rs;
    }
    return recordSource;
}

Example 2 with VirtualColumn

use of com.questdb.ql.ops.VirtualColumn in project questdb by bluestreak01.

the class QueryCompiler method compileJournal0.

@SuppressWarnings("ConstantConditions")
private RecordSource compileJournal0(QueryModel model, Factory factory) throws ParserException {
    applyLimit(model);
    RecordMetadata metadata = model.getMetadata();
    JournalMetadata journalMetadata;
    if (metadata == null) {
        metadata = model.collectJournalMetadata(factory);
    }
    if (metadata instanceof JournalMetadata) {
        journalMetadata = (JournalMetadata) metadata;
    } else {
        throw QueryError.$(0, "Internal error: invalid metadata");
    }
    if (model.getAlias() != null) {
        journalMetadata.setAlias(model.getAlias().token);
    } else {
        journalMetadata.setAlias(QueryModel.stripMarker(model.getJournalName().token));
    }
    PartitionSource ps = new JournalPartitionSource(journalMetadata, true);
    RowSource rs = null;
    String latestByCol = null;
    RecordColumnMetadata latestByMetadata = null;
    ExprNode latestByNode = null;
    if (model.getLatestBy() != null) {
        latestByNode = model.getLatestBy();
        if (latestByNode.type != ExprNode.LITERAL) {
            throw QueryError.$(latestByNode.position, "Column name expected");
        }
        latestByCol = model.translateAlias(latestByNode.token).toString();
        int colIndex = journalMetadata.getColumnIndexQuiet(latestByCol);
        if (colIndex == -1) {
            throw QueryError.invalidColumn(latestByNode.position, latestByNode.token);
        }
        latestByMetadata = journalMetadata.getColumnQuick(colIndex);
        int type = latestByMetadata.getType();
        if (type != ColumnType.SYMBOL && type != ColumnType.STRING && type != ColumnType.INT && type != ColumnType.LONG) {
            throw QueryError.position(latestByNode.position).$("Expected symbol, string, int or long column, found: ").$(ColumnType.nameOf(type)).$();
        }
        if (!latestByMetadata.isIndexed()) {
            throw QueryError.position(latestByNode.position).$("Column is not indexed").$();
        }
    }
    ExprNode where = model.getWhereClause();
    if (where != null) {
        IntrinsicModel im = queryFilterAnalyser.extract(model, where, journalMetadata, latestByCol, getTimestampIndexQuiet(model.getTimestamp(), journalMetadata));
        VirtualColumn filter = im.filter != null ? virtualColumnBuilder.createVirtualColumn(model, im.filter, journalMetadata) : null;
        if (filter != null) {
            if (filter.getType() != ColumnType.BOOLEAN) {
                throw QueryError.$(im.filter.position, "Boolean expression expected");
            }
            if (filter.isConstant()) {
                if (filter.getBool(null)) {
                    // constant TRUE, no filtering needed
                    filter = null;
                } else {
                    im.intrinsicValue = IntrinsicValue.FALSE;
                }
            }
        }
        if (im.intrinsicValue == IntrinsicValue.FALSE) {
            ps = new NoOpJournalPartitionSource(journalMetadata);
        } else {
            if (im.intervals != null) {
                ps = new MultiIntervalPartitionSource(ps, im.intervals);
            }
            if (latestByCol == null) {
                if (im.keyColumn != null) {
                    switch(journalMetadata.getColumn(im.keyColumn).getType()) {
                        case ColumnType.SYMBOL:
                            rs = buildRowSourceForSym(im);
                            break;
                        case ColumnType.STRING:
                            rs = buildRowSourceForStr(im);
                            break;
                        case ColumnType.INT:
                            rs = buildRowSourceForInt(im);
                            break;
                        case ColumnType.LONG:
                            rs = buildRowSourceForLong(im);
                            break;
                        default:
                            break;
                    }
                }
                if (filter != null) {
                    rs = new FilteredRowSource(rs == null ? new AllRowSource() : rs, filter);
                }
            } else {
                if (im.keyColumn != null && im.keyValuesIsLambda) {
                    int lambdaColIndex;
                    RecordSource lambda = compileSourceInternal(factory, im.keyValues.get(0));
                    RecordMetadata m = lambda.getMetadata();
                    switch(m.getColumnCount()) {
                        case 0:
                            Misc.free(lambda);
                            throw QueryError.$(im.keyValuePositions.getQuick(0), "Query must select at least one column");
                        case 1:
                            lambdaColIndex = 0;
                            break;
                        default:
                            lambdaColIndex = m.getColumnIndexQuiet(latestByCol);
                            if (lambdaColIndex == -1) {
                                Misc.free(lambda);
                                throw QueryError.$(im.keyValuePositions.getQuick(0), "Ambiguous column names in lambda query. Specify select clause");
                            }
                            break;
                    }
                    int lambdaColType = m.getColumnQuick(lambdaColIndex).getType();
                    mutableSig.setParamCount(2).setName("").paramType(0, latestByMetadata.getType(), true).paramType(1, lambdaColType, false);
                    LatestByLambdaRowSourceFactory fact = LAMBDA_ROW_SOURCE_FACTORIES.get(mutableSig);
                    if (fact != null) {
                        rs = fact.newInstance(latestByCol, lambda, lambdaColIndex, filter);
                    } else {
                        Misc.free(lambda);
                        throw QueryError.$(im.keyValuePositions.getQuick(0), "Mismatched types");
                    }
                } else {
                    switch(latestByMetadata.getType()) {
                        case ColumnType.SYMBOL:
                            if (im.keyColumn != null) {
                                rs = new KvIndexSymListHeadRowSource(latestByCol, new CharSequenceHashSet(im.keyValues), filter);
                            } else {
                                rs = new KvIndexSymAllHeadRowSource(latestByCol, filter);
                            }
                            break;
                        case ColumnType.STRING:
                            if (im.keyColumn != null) {
                                rs = new KvIndexStrListHeadRowSource(latestByCol, new CharSequenceHashSet(im.keyValues), filter);
                            } else {
                                Misc.free(rs);
                                throw QueryError.$(latestByNode.position, "Filter on string column expected");
                            }
                            break;
                        case ColumnType.LONG:
                            if (im.keyColumn != null) {
                                rs = new KvIndexLongListHeadRowSource(latestByCol, toLongHashSet(im), filter);
                            } else {
                                Misc.free(rs);
                                throw QueryError.$(latestByNode.position, "Filter on long column expected");
                            }
                            break;
                        case ColumnType.INT:
                            if (im.keyColumn != null) {
                                rs = new KvIndexIntListHeadRowSource(latestByCol, toIntHashSet(im), filter);
                            } else {
                                Misc.free(rs);
                                throw QueryError.$(latestByNode.position, "Filter on int column expected");
                            }
                            break;
                        default:
                            break;
                    }
                }
            }
        }
    } else if (latestByCol != null) {
        switch(latestByMetadata.getType()) {
            case ColumnType.SYMBOL:
                rs = new KvIndexSymAllHeadRowSource(latestByCol, null);
                break;
            default:
                Misc.free(rs);
                throw QueryError.$(latestByNode.position, "Only SYM columns can be used here without filter");
        }
    }
    // check for case of simple "select count() from tab"
    if (rs == null && model.getColumns().size() == 1) {
        QueryColumn qc = model.getColumns().getQuick(0);
        if ("count".equals(qc.getAst().token) && qc.getAst().paramCount == 0) {
            // remove order clause
            model.getOrderBy().clear();
            // remove columns so that there is no wrapping of result source
            model.getColumns().clear();
            return new CountRecordSource(qc.getAlias() == null ? "count" : qc.getAlias(), ps);
        }
    }
    RecordSource recordSource = new JournalRecordSource(ps, rs == null ? new AllRowSource() : rs);
    if (QueryModel.hasMarker(model.getJournalName().token)) {
        return new NoRowIdRecordSource().of(recordSource);
    }
    return recordSource;
}

Example 3 with VirtualColumn

use of com.questdb.ql.ops.VirtualColumn in project questdb by bluestreak01.

the class QueryCompiler method compileOuterVirtualColumns.

private RecordSource compileOuterVirtualColumns(RecordSource rs, QueryModel model) throws ParserException {
    ObjList<VirtualColumn> outer = new ObjList<>(outerVirtualColumns.size());
    for (int i = 0, n = outerVirtualColumns.size(); i < n; i++) {
        QueryColumn qc = outerVirtualColumns.get(i);
        VirtualColumn vc = virtualColumnBuilder.createVirtualColumn(model, qc.getAst(), rs.getMetadata());
        vc.setName(qc.getAlias());
        outer.add(vc);
    }
    return new VirtualColumnRecordSource(rs, outer);
}

Example 4 with VirtualColumn

use of com.questdb.ql.ops.VirtualColumn in project questdb by bluestreak01.

the class QueryCompiler method filter.

private RecordSource filter(QueryModel model, RecordSource rs) throws ParserException {
    try {
        if (model.getWhereClause() == null) {
            return rs;
        }
        RecordMetadata m = rs.getMetadata();
        if (model.getAlias() != null) {
            m.setAlias(model.getAlias().token);
        }
        int timestampIndex = getTimestampIndexQuiet(model.getTimestamp(), m);
        IntrinsicModel im = queryFilterAnalyser.extract(model, model.getWhereClause(), m, null, timestampIndex);
        if (im.intrinsicValue == IntrinsicValue.FALSE) {
            return new NoOpJournalRecordSource(rs);
        }
        if (im.intervals != null) {
            rs = new IntervalRecordSource(rs, im.intervals, timestampIndex);
        }
        if (im.filter != null) {
            VirtualColumn vc = virtualColumnBuilder.createVirtualColumn(model, im.filter, m);
            if (vc.isConstant()) {
                // todo: not hit by test
                if (vc.getBool(null)) {
                    return rs;
                } else {
                    return new NoOpJournalRecordSource(rs);
                }
            }
            return new FilteredRecordSource(rs, vc, im.filter);
        } else {
            return rs;
        }
    } catch (ParserException e) {
        Misc.free(rs);
        throw e;
    }
}

Example 5 with VirtualColumn

use of com.questdb.ql.ops.VirtualColumn in project questdb by bluestreak01.

the class QueryCompiler method compileAggregates.

private RecordSource compileAggregates(RecordSource rs, QueryModel model) throws ParserException {
    final int n = aggregators.size();
    final ExprNode sampleBy = model.getSampleBy();
    ObjList<AggregatorFunction> af = new ObjList<>(n);
    // create virtual columns
    for (int i = 0; i < n; i++) {
        QueryColumn qc = aggregators.get(i);
        VirtualColumn vc = virtualColumnBuilder.createVirtualColumn(model, qc.getAst(), rs.getMetadata());
        if (vc instanceof AggregatorFunction) {
            vc.setName(qc.getAlias());
            af.add((AggregatorFunction) vc);
        } else {
            throw QueryError.$(qc.getAst().position, "Internal configuration error. Not an aggregate");
        }
    }
    RecordSource out;
    if (sampleBy == null) {
        out = new AggregatedRecordSource(rs, groupKeyColumns, af, configuration.getDbAggregatePage(), recordKeyCopierCompiler);
    } else {
        TimestampSampler sampler = SamplerFactory.from(sampleBy.token);
        if (sampler == null) {
            throw QueryError.$(sampleBy.position, "Invalid sample");
        }
        out = new ResampledRecordSource(rs, getTimestampIndex(model, model.getTimestamp(), rs.getMetadata()), groupKeyColumns, af, sampler, configuration.getDbAggregatePage(), recordKeyCopierCompiler);
    }
    return out;
}