Examples with EmptyTableRecordCursorFactory io.questdb.griffin.engine.EmptyTableRecordCursorFactory used on opensource projects

Example 1 with EmptyTableRecordCursorFactory

use of io.questdb.griffin.engine.EmptyTableRecordCursorFactory in project questdb by bluestreak01.

the class SqlCodeGenerator method generateTableQuery.

private RecordCursorFactory generateTableQuery(QueryModel model, SqlExecutionContext executionContext) throws SqlException {
    final ObjList<ExpressionNode> latestBy = model.getLatestBy();
    try (TableReader reader = engine.getReader(executionContext.getCairoSecurityContext(), model.getTableName().token, model.getTableId(), model.getTableVersion())) {
        final RecordMetadata readerMeta = reader.getMetadata();
        // create metadata based on top-down columns that are required
        final ObjList<QueryColumn> topDownColumns = model.getTopDownColumns();
        final int topDownColumnCount = topDownColumns.size();
        final IntList columnIndexes = new IntList();
        final IntList columnSizes = new IntList();
        // topDownColumnCount can be 0 for 'select count()' queries
        int readerTimestampIndex;
        try {
            readerTimestampIndex = getTimestampIndex(model, readerMeta);
        } catch (SqlException e) {
            Misc.free(reader);
            throw e;
        }
        boolean requiresTimestamp = joinsRequiringTimestamp[model.getJoinType()];
        final GenericRecordMetadata myMeta = new GenericRecordMetadata();
        boolean framingSupported;
        try {
            if (requiresTimestamp) {
                executionContext.pushTimestampRequiredFlag(true);
            }
            boolean contextTimestampRequired = executionContext.isTimestampRequired();
            // for example "select count() from x sample by 1h" implicitly needs timestamp column selected
            if (topDownColumnCount > 0 || contextTimestampRequired) {
                framingSupported = true;
                for (int i = 0; i < topDownColumnCount; i++) {
                    int columnIndex = readerMeta.getColumnIndexQuiet(topDownColumns.getQuick(i).getName());
                    int type = readerMeta.getColumnType(columnIndex);
                    int typeSize = ColumnType.sizeOf(type);
                    // if (framingSupported && (typeSize < Byte.BYTES || typeSize > Double.BYTES)) {
                    // we don't frame non-primitive types yet
                    // framingSupported = false;
                    // }
                    columnIndexes.add(columnIndex);
                    columnSizes.add((Numbers.msb(typeSize)));
                    myMeta.add(new TableColumnMetadata(Chars.toString(topDownColumns.getQuick(i).getName()), readerMeta.getColumnHash(columnIndex), type, readerMeta.isColumnIndexed(columnIndex), readerMeta.getIndexValueBlockCapacity(columnIndex), readerMeta.isSymbolTableStatic(columnIndex), readerMeta.getMetadata(columnIndex)));
                    if (columnIndex == readerTimestampIndex) {
                        myMeta.setTimestampIndex(myMeta.getColumnCount() - 1);
                    }
                }
                // select timestamp when it is required but not already selected
                if (readerTimestampIndex != -1 && myMeta.getTimestampIndex() == -1 && contextTimestampRequired) {
                    myMeta.add(new TableColumnMetadata(readerMeta.getColumnName(readerTimestampIndex), readerMeta.getColumnHash(readerTimestampIndex), readerMeta.getColumnType(readerTimestampIndex), readerMeta.getMetadata(readerTimestampIndex)));
                    myMeta.setTimestampIndex(myMeta.getColumnCount() - 1);
                    columnIndexes.add(readerTimestampIndex);
                    columnSizes.add((Numbers.msb(ColumnType.TIMESTAMP)));
                }
            } else {
                framingSupported = false;
            }
        } finally {
            if (requiresTimestamp) {
                executionContext.popTimestampRequiredFlag();
            }
        }
        listColumnFilterA.clear();
        final int latestByColumnCount = latestBy.size();
        if (latestByColumnCount > 0) {
            // first check if column is valid
            for (int i = 0; i < latestByColumnCount; i++) {
                final int index = myMeta.getColumnIndexQuiet(latestBy.getQuick(i).token);
                if (index == -1) {
                    throw SqlException.invalidColumn(latestBy.getQuick(i).position, latestBy.getQuick(i).token);
                }
                // we are reusing collections which leads to confusing naming for this method
                // keyTypes are types of columns we collect 'latest by' for
                keyTypes.add(myMeta.getColumnType(index));
                // columnFilterA are indexes of columns we collect 'latest by' for
                listColumnFilterA.add(index + 1);
            }
        }
        final String tableName = reader.getTableName();
        final ExpressionNode withinExtracted = whereClauseParser.extractWithin(model, model.getWhereClause(), readerMeta, functionParser, executionContext, prefixes);
        model.setWhereClause(withinExtracted);
        if (withinExtracted != null) {
            CharSequence preferredKeyColumn = null;
            if (listColumnFilterA.size() == 1) {
                final int latestByIndex = listColumnFilterA.getColumnIndexFactored(0);
                if (ColumnType.isSymbol(myMeta.getColumnType(latestByIndex))) {
                    preferredKeyColumn = latestBy.getQuick(0).token;
                }
            }
            final IntrinsicModel intrinsicModel = whereClauseParser.extract(model, withinExtracted, readerMeta, preferredKeyColumn, readerTimestampIndex, functionParser, myMeta, executionContext);
            // intrinsic parser can collapse where clause when removing parts it can replace
            // need to make sure that filter is updated on the model in case it is processed up the call stack
            // 
            // At this juncture filter can use used up by one of the implementations below.
            // We will clear it preemptively. If nothing picks filter up we will set model "where"
            // to the downsized filter
            model.setWhereClause(null);
            if (intrinsicModel.intrinsicValue == IntrinsicModel.FALSE) {
                return new EmptyTableRecordCursorFactory(myMeta);
            }
            DataFrameCursorFactory dfcFactory;
            if (latestByColumnCount > 0) {
                Function f = compileFilter(intrinsicModel, myMeta, executionContext);
                if (f != null && f.isConstant() && !f.getBool(null)) {
                    return new EmptyTableRecordCursorFactory(myMeta);
                }
                return generateLatestByQuery(model, reader, myMeta, tableName, intrinsicModel, f, executionContext, readerTimestampIndex, columnIndexes, prefixes);
            }
            // below code block generates index-based filter
            final boolean intervalHitsOnlyOnePartition;
            if (intrinsicModel.hasIntervalFilters()) {
                RuntimeIntrinsicIntervalModel intervalModel = intrinsicModel.buildIntervalModel();
                dfcFactory = new IntervalFwdDataFrameCursorFactory(engine, tableName, model.getTableId(), model.getTableVersion(), intervalModel, readerTimestampIndex);
                intervalHitsOnlyOnePartition = intervalModel.allIntervalsHitOnePartition(reader.getPartitionedBy());
            } else {
                dfcFactory = new FullFwdDataFrameCursorFactory(engine, tableName, model.getTableId(), model.getTableVersion());
                intervalHitsOnlyOnePartition = false;
            }
            if (intrinsicModel.keyColumn != null) {
                // existence of column would have been already validated
                final int keyColumnIndex = reader.getMetadata().getColumnIndexQuiet(intrinsicModel.keyColumn);
                final int nKeyValues = intrinsicModel.keyValues.size();
                final int nKeyExcludedValues = intrinsicModel.keyExcludedValues.size();
                if (intrinsicModel.keySubQuery != null) {
                    final RecordCursorFactory rcf = generate(intrinsicModel.keySubQuery, executionContext);
                    final Record.CharSequenceFunction func = validateSubQueryColumnAndGetGetter(intrinsicModel, rcf.getMetadata());
                    Function f = compileFilter(intrinsicModel, myMeta, executionContext);
                    if (f != null && f.isConstant() && !f.getBool(null)) {
                        return new EmptyTableRecordCursorFactory(myMeta);
                    }
                    return new FilterOnSubQueryRecordCursorFactory(myMeta, dfcFactory, rcf, keyColumnIndex, f, func, columnIndexes);
                }
                assert nKeyValues > 0 || nKeyExcludedValues > 0;
                boolean orderByKeyColumn = false;
                int indexDirection = BitmapIndexReader.DIR_FORWARD;
                if (intervalHitsOnlyOnePartition) {
                    final ObjList<ExpressionNode> orderByAdvice = model.getOrderByAdvice();
                    final int orderByAdviceSize = orderByAdvice.size();
                    if (orderByAdviceSize > 0 && orderByAdviceSize < 3) {
                        // "overhead" order by implementation, which would be trying to oder already ordered symbols
                        if (Chars.equals(orderByAdvice.getQuick(0).token, intrinsicModel.keyColumn)) {
                            myMeta.setTimestampIndex(-1);
                            if (orderByAdviceSize == 1) {
                                orderByKeyColumn = true;
                            } else if (Chars.equals(orderByAdvice.getQuick(1).token, model.getTimestamp().token)) {
                                orderByKeyColumn = true;
                                if (getOrderByDirectionOrDefault(model, 1) == QueryModel.ORDER_DIRECTION_DESCENDING) {
                                    indexDirection = BitmapIndexReader.DIR_BACKWARD;
                                }
                            }
                        }
                    }
                }
                if (intrinsicModel.keyExcludedValues.size() == 0) {
                    Function f = compileFilter(intrinsicModel, myMeta, executionContext);
                    if (f != null && f.isConstant()) {
                        try {
                            if (!f.getBool(null)) {
                                return new EmptyTableRecordCursorFactory(myMeta);
                            }
                        } finally {
                            f = Misc.free(f);
                        }
                    }
                    if (nKeyValues == 1) {
                        final RowCursorFactory rcf;
                        final CharSequence symbol = intrinsicModel.keyValues.get(0);
                        final int symbolKey = reader.getSymbolMapReader(keyColumnIndex).keyOf(symbol);
                        if (symbolKey == SymbolTable.VALUE_NOT_FOUND) {
                            if (f == null) {
                                rcf = new DeferredSymbolIndexRowCursorFactory(keyColumnIndex, functionParser.createBindVariable(intrinsicModel.keyValuePositions.getQuick(0), symbol), true, indexDirection);
                            } else {
                                rcf = new DeferredSymbolIndexFilteredRowCursorFactory(keyColumnIndex, functionParser.createBindVariable(intrinsicModel.keyValuePositions.getQuick(0), symbol), f, true, indexDirection, columnIndexes);
                            }
                        } else {
                            if (f == null) {
                                rcf = new SymbolIndexRowCursorFactory(keyColumnIndex, symbolKey, true, indexDirection, null);
                            } else {
                                rcf = new SymbolIndexFilteredRowCursorFactory(keyColumnIndex, symbolKey, f, true, indexDirection, columnIndexes, null);
                            }
                        }
                        if (f == null) {
                            // cursors in Sample By processing
                            return new DeferredSingleSymbolFilterDataFrameRecordCursorFactory(keyColumnIndex, symbol, rcf, myMeta, dfcFactory, orderByKeyColumn, columnIndexes, columnSizes);
                        }
                        return new DataFrameRecordCursorFactory(myMeta, dfcFactory, rcf, orderByKeyColumn, f, false, columnIndexes, columnSizes);
                    }
                    symbolValueList.clear();
                    for (int i = 0, n = intrinsicModel.keyValues.size(); i < n; i++) {
                        symbolValueList.add(functionParser.createBindVariable(intrinsicModel.keyValuePositions.getQuick(i), intrinsicModel.keyValues.get(i)));
                    }
                    if (orderByKeyColumn) {
                        myMeta.setTimestampIndex(-1);
                    }
                    return new FilterOnValuesRecordCursorFactory(myMeta, dfcFactory, symbolValueList, keyColumnIndex, reader, f, model.getOrderByAdviceMnemonic(), orderByKeyColumn, getOrderByDirectionOrDefault(model, 0), indexDirection, columnIndexes);
                } else if (intrinsicModel.keyExcludedValues.size() > 0 && reader.getSymbolMapReader(keyColumnIndex).size() < configuration.getMaxSymbolNotEqualsCount()) {
                    symbolValueList.clear();
                    for (int i = 0, n = intrinsicModel.keyExcludedValues.size(); i < n; i++) {
                        symbolValueList.add(functionParser.createBindVariable(intrinsicModel.keyExcludedValuePositions.getQuick(i), intrinsicModel.keyExcludedValues.get(i)));
                    }
                    Function f = compileFilter(intrinsicModel, myMeta, executionContext);
                    if (f != null && f.isConstant()) {
                        try {
                            if (!f.getBool(null)) {
                                return new EmptyTableRecordCursorFactory(myMeta);
                            }
                        } finally {
                            f = Misc.free(f);
                        }
                    }
                    return new FilterOnExcludedValuesRecordCursorFactory(myMeta, dfcFactory, symbolValueList, keyColumnIndex, f, model.getOrderByAdviceMnemonic(), orderByKeyColumn, indexDirection, columnIndexes, configuration.getMaxSymbolNotEqualsCount());
                }
            }
            if (intervalHitsOnlyOnePartition && intrinsicModel.filter == null) {
                final ObjList<ExpressionNode> orderByAdvice = model.getOrderByAdvice();
                final int orderByAdviceSize = orderByAdvice.size();
                if (orderByAdviceSize > 0 && orderByAdviceSize < 3 && intrinsicModel.hasIntervalFilters()) {
                    // we can only deal with 'order by symbol, timestamp' at best
                    // skip this optimisation if order by is more extensive
                    final int columnIndex = myMeta.getColumnIndexQuiet(model.getOrderByAdvice().getQuick(0).token);
                    assert columnIndex > -1;
                    // this is our kind of column
                    if (myMeta.isColumnIndexed(columnIndex)) {
                        boolean orderByKeyColumn = false;
                        int indexDirection = BitmapIndexReader.DIR_FORWARD;
                        if (orderByAdviceSize == 1) {
                            orderByKeyColumn = true;
                        } else if (Chars.equals(orderByAdvice.getQuick(1).token, model.getTimestamp().token)) {
                            orderByKeyColumn = true;
                            if (getOrderByDirectionOrDefault(model, 1) == QueryModel.ORDER_DIRECTION_DESCENDING) {
                                indexDirection = BitmapIndexReader.DIR_BACKWARD;
                            }
                        }
                        if (orderByKeyColumn) {
                            // check that intrinsicModel.intervals hit only one partition
                            myMeta.setTimestampIndex(-1);
                            return new SortedSymbolIndexRecordCursorFactory(myMeta, dfcFactory, columnIndex, getOrderByDirectionOrDefault(model, 0) == QueryModel.ORDER_DIRECTION_ASCENDING, indexDirection, columnIndexes);
                        }
                    }
                }
            }
            model.setWhereClause(intrinsicModel.filter);
            return new DataFrameRecordCursorFactory(myMeta, dfcFactory, new DataFrameRowCursorFactory(), false, null, framingSupported, columnIndexes, columnSizes);
        }
        // no where clause
        if (latestByColumnCount == 0) {
            return new DataFrameRecordCursorFactory(myMeta, new FullFwdDataFrameCursorFactory(engine, tableName, model.getTableId(), model.getTableVersion()), new DataFrameRowCursorFactory(), false, null, framingSupported, columnIndexes, columnSizes);
        }
        if (latestByColumnCount == 1 && myMeta.isColumnIndexed(listColumnFilterA.getColumnIndexFactored(0))) {
            return new LatestByAllIndexedFilteredRecordCursorFactory(configuration, myMeta, new FullBwdDataFrameCursorFactory(engine, tableName, model.getTableId(), model.getTableVersion()), listColumnFilterA.getColumnIndexFactored(0), null, columnIndexes, prefixes);
        }
        return new LatestByAllFilteredRecordCursorFactory(myMeta, configuration, new FullBwdDataFrameCursorFactory(engine, tableName, model.getTableId(), model.getTableVersion()), RecordSinkFactory.getInstance(asm, myMeta, listColumnFilterA, false), keyTypes, null, columnIndexes);
    }
}

Example 2 with EmptyTableRecordCursorFactory

use of io.questdb.griffin.engine.EmptyTableRecordCursorFactory in project questdb by bluestreak01.

the class SqlCodeGenerator method generateFilter0.

@NotNull
private RecordCursorFactory generateFilter0(RecordCursorFactory factory, QueryModel model, SqlExecutionContext executionContext, ExpressionNode filter) throws SqlException {
    model.setWhereClause(null);
    final Function f = compileFilter(filter, factory.getMetadata(), executionContext);
    if (f.isConstant()) {
        // noinspection TryFinallyCanBeTryWithResources
        try {
            if (f.getBool(null)) {
                return factory;
            }
            // metadata is always a GenericRecordMetadata instance
            return new EmptyTableRecordCursorFactory(factory.getMetadata());
        } finally {
            f.close();
        }
    }
    return new FilteredRecordCursorFactory(factory, f);
}

Example 3 with EmptyTableRecordCursorFactory

use of io.questdb.griffin.engine.EmptyTableRecordCursorFactory in project questdb by bluestreak01.

the class SqlCodeGenerator method generateJoins.

private RecordCursorFactory generateJoins(QueryModel model, SqlExecutionContext executionContext) throws SqlException {
    final ObjList<QueryModel> joinModels = model.getJoinModels();
    IntList ordered = model.getOrderedJoinModels();
    RecordCursorFactory master = null;
    CharSequence masterAlias = null;
    try {
        int n = ordered.size();
        assert n > 1;
        for (int i = 0; i < n; i++) {
            int index = ordered.getQuick(i);
            QueryModel slaveModel = joinModels.getQuick(index);
            if (i > 0) {
                executionContext.pushTimestampRequiredFlag(joinsRequiringTimestamp[slaveModel.getJoinType()]);
            } else {
                // i == 0
                // This is first model in the sequence of joins
                // TS requirement is symmetrical on both right and left sides
                // check if next join requires a timestamp
                int nextJointType = joinModels.getQuick(ordered.getQuick(1)).getJoinType();
                executionContext.pushTimestampRequiredFlag(joinsRequiringTimestamp[nextJointType]);
            }
            RecordCursorFactory slave = null;
            try {
                // compile
                slave = generateQuery(slaveModel, executionContext, index > 0);
                // check if this is the root of joins
                if (master == null) {
                    // This is an opportunistic check of order by clause
                    // to determine if we can get away ordering main record source only
                    // Ordering main record source could benefit from rowid access thus
                    // making it faster compared to ordering of join record source that
                    // doesn't allow rowid access.
                    master = slave;
                    masterAlias = slaveModel.getName();
                } else {
                    // not the root, join to "master"
                    final int joinType = slaveModel.getJoinType();
                    final RecordMetadata masterMetadata = master.getMetadata();
                    final RecordMetadata slaveMetadata = slave.getMetadata();
                    switch(joinType) {
                        case JOIN_CROSS:
                            master = new CrossJoinRecordCursorFactory(createJoinMetadata(masterAlias, masterMetadata, slaveModel.getName(), slaveMetadata), master, slave, masterMetadata.getColumnCount());
                            masterAlias = null;
                            break;
                        case JOIN_ASOF:
                            validateBothTimestamps(slaveModel, masterMetadata, slaveMetadata);
                            processJoinContext(index == 1, slaveModel.getContext(), masterMetadata, slaveMetadata);
                            if (slave.recordCursorSupportsRandomAccess() && !fullFatJoins) {
                                if (listColumnFilterA.size() > 0 && listColumnFilterB.size() > 0) {
                                    master = createAsOfJoin(createJoinMetadata(masterAlias, masterMetadata, slaveModel.getName(), slaveMetadata), master, RecordSinkFactory.getInstance(asm, masterMetadata, listColumnFilterB, true), slave, RecordSinkFactory.getInstance(asm, slaveMetadata, listColumnFilterA, true), masterMetadata.getColumnCount());
                                } else {
                                    master = new AsOfJoinNoKeyRecordCursorFactory(createJoinMetadata(masterAlias, masterMetadata, slaveModel.getName(), slaveMetadata), master, slave, masterMetadata.getColumnCount());
                                }
                            } else {
                                master = createFullFatJoin(master, masterMetadata, masterAlias, slave, slaveMetadata, slaveModel.getName(), slaveModel.getJoinKeywordPosition(), CREATE_FULL_FAT_AS_OF_JOIN);
                            }
                            masterAlias = null;
                            break;
                        case JOIN_LT:
                            validateBothTimestamps(slaveModel, masterMetadata, slaveMetadata);
                            processJoinContext(index == 1, slaveModel.getContext(), masterMetadata, slaveMetadata);
                            if (slave.recordCursorSupportsRandomAccess() && !fullFatJoins) {
                                if (listColumnFilterA.size() > 0 && listColumnFilterB.size() > 0) {
                                    master = createLtJoin(createJoinMetadata(masterAlias, masterMetadata, slaveModel.getName(), slaveMetadata), master, RecordSinkFactory.getInstance(asm, masterMetadata, listColumnFilterB, true), slave, RecordSinkFactory.getInstance(asm, slaveMetadata, listColumnFilterA, true), masterMetadata.getColumnCount());
                                } else {
                                    master = new LtJoinNoKeyRecordCursorFactory(createJoinMetadata(masterAlias, masterMetadata, slaveModel.getName(), slaveMetadata), master, slave, masterMetadata.getColumnCount());
                                }
                            } else {
                                master = createFullFatJoin(master, masterMetadata, masterAlias, slave, slaveMetadata, slaveModel.getName(), slaveModel.getJoinKeywordPosition(), CREATE_FULL_FAT_LT_JOIN);
                            }
                            masterAlias = null;
                            break;
                        case JOIN_SPLICE:
                            validateBothTimestamps(slaveModel, masterMetadata, slaveMetadata);
                            processJoinContext(index == 1, slaveModel.getContext(), masterMetadata, slaveMetadata);
                            if (slave.recordCursorSupportsRandomAccess() && master.recordCursorSupportsRandomAccess() && !fullFatJoins) {
                                master = createSpliceJoin(// splice join result does not have timestamp
                                createJoinMetadata(masterAlias, masterMetadata, slaveModel.getName(), slaveMetadata, -1), master, RecordSinkFactory.getInstance(asm, masterMetadata, listColumnFilterB, true), slave, RecordSinkFactory.getInstance(asm, slaveMetadata, listColumnFilterA, true), masterMetadata.getColumnCount());
                            } else {
                                assert false;
                            }
                            break;
                        default:
                            processJoinContext(index == 1, slaveModel.getContext(), masterMetadata, slaveMetadata);
                            master = createHashJoin(createJoinMetadata(masterAlias, masterMetadata, slaveModel.getName(), slaveMetadata), master, slave, joinType);
                            masterAlias = null;
                            break;
                    }
                }
            } catch (Throwable th) {
                Misc.free(master);
                Misc.free(slave);
                throw th;
            } finally {
                executionContext.popTimestampRequiredFlag();
            }
            // check if there are post-filters
            ExpressionNode filter = slaveModel.getPostJoinWhereClause();
            if (filter != null) {
                master = new FilteredRecordCursorFactory(master, functionParser.parseFunction(filter, master.getMetadata(), executionContext));
            }
        }
        // unfortunately we had to go all out to create join metadata
        // now it is time to check if we have constant conditions
        ExpressionNode constFilter = model.getConstWhereClause();
        if (constFilter != null) {
            Function function = functionParser.parseFunction(constFilter, null, executionContext);
            if (!function.getBool(null)) {
                // do not copy metadata here
                // this would have been JoinRecordMetadata, which is new instance anyway
                // we have to make sure that this metadata is safely transitioned
                // to empty cursor factory
                JoinRecordMetadata metadata = (JoinRecordMetadata) master.getMetadata();
                metadata.incrementRefCount();
                RecordCursorFactory factory = new EmptyTableRecordCursorFactory(metadata);
                Misc.free(master);
                return factory;
            }
        }
        return master;
    } catch (Throwable e) {
        Misc.free(master);
        throw e;
    }
}