Examples with ScanLimits herddb.model.ScanLimits used on opensource projects

Example 1 with ScanLimits

use of herddb.model.ScanLimits in project herddb by diennea.

the class TableManager method scanWithStream.

private DataScanner scanWithStream(ScanStatement statement, StatementEvaluationContext context, Transaction transaction, boolean lockRequired, boolean forWrite) throws StatementExecutionException {
    if (transaction != null) {
        transaction.increaseRefcount();
    }
    try {
        final TupleComparator comparator = statement.getComparator();
        boolean sorted = comparator != null;
        boolean sortedByClusteredIndex = comparator != null && comparator.isOnlyPrimaryKeyAndAscending() && keyToPageSortedAscending;
        final Projection projection = statement.getProjection();
        final boolean applyProjectionDuringScan = projection != null && !sorted;
        ScanLimits limits = statement.getLimits();
        int maxRows = limits == null ? 0 : limits.computeMaxRows(context);
        int offset = limits == null ? 0 : limits.computeOffset(context);
        Stream<DataAccessor> result;
        Function<Record, DataAccessor> mapper = (Record record) -> {
            DataAccessor tuple;
            if (applyProjectionDuringScan) {
                tuple = projection.map(record.getDataAccessor(table), context);
            } else {
                tuple = record.getDataAccessor(table);
            }
            return tuple;
        };
        Stream<Record> recordsFromTransactionSorted = streamTransactionData(transaction, statement.getPredicate(), context);
        Stream<DataAccessor> fromTransactionSorted = recordsFromTransactionSorted != null ? recordsFromTransactionSorted.map(mapper) : null;
        if (fromTransactionSorted != null && comparator != null) {
            fromTransactionSorted = fromTransactionSorted.sorted(comparator);
        }
        Stream<DataAccessor> tableData = streamTableData(statement, context, transaction, lockRequired, forWrite).map(mapper);
        if (maxRows > 0) {
            if (sortedByClusteredIndex) {
                // already sorted if needed
                if (fromTransactionSorted != null) {
                    // already sorted from index
                    tableData = tableData.limit(maxRows + offset);
                    fromTransactionSorted = fromTransactionSorted.limit(maxRows + offset);
                    // we need to re-sort after merging the data
                    result = Stream.concat(fromTransactionSorted, tableData).sorted(comparator);
                } else {
                    // already sorted from index
                    tableData = tableData.limit(maxRows + offset);
                    // no need to re-sort
                    result = tableData;
                }
            } else if (sorted) {
                // need to sort
                tableData = tableData.sorted(comparator);
                // already sorted if needed
                if (fromTransactionSorted != null) {
                    tableData = tableData.limit(maxRows + offset);
                    fromTransactionSorted = fromTransactionSorted.limit(maxRows + offset);
                    // we need to re-sort after merging the data
                    result = Stream.concat(fromTransactionSorted, tableData).sorted(comparator);
                } else {
                    tableData = tableData.limit(maxRows + offset);
                    // no need to sort again
                    result = tableData;
                }
            } else if (fromTransactionSorted == null) {
                result = tableData;
            } else {
                result = Stream.concat(fromTransactionSorted, tableData);
            }
        } else {
            if (sortedByClusteredIndex) {
                // already sorted from index
                if (fromTransactionSorted != null) {
                    tableData = tableData.sorted(comparator);
                    // fromTransactionSorted is already sorted
                    // we need to re-sort
                    result = Stream.concat(fromTransactionSorted, tableData).sorted(comparator);
                } else {
                    result = tableData;
                }
            } else if (sorted) {
                // we need to re-sort
                if (fromTransactionSorted != null) {
                    result = Stream.concat(fromTransactionSorted, tableData).sorted(comparator);
                } else {
                    result = tableData.sorted(comparator);
                }
            } else if (fromTransactionSorted != null) {
                // no need to sort
                result = Stream.concat(fromTransactionSorted, tableData);
            } else {
                result = tableData;
            }
        }
        if (offset > 0) {
            result = result.skip(offset);
        }
        if (maxRows > 0) {
            result = result.limit(maxRows);
        }
        if (!applyProjectionDuringScan && projection != null) {
            result = result.map(r -> projection.map(r, context));
        }
        String[] fieldNames;
        Column[] columns;
        if (projection != null) {
            fieldNames = projection.getFieldNames();
            columns = projection.getColumns();
        } else {
            fieldNames = table.columnNames;
            columns = table.columns;
        }
        return new StreamDataScanner(transaction, fieldNames, columns, result);
    } finally {
        if (transaction != null) {
            transaction.decreaseRefCount();
        }
    }
}

Example 2 with ScanLimits

use of herddb.model.ScanLimits in project herddb by diennea.

the class TableManager method scanNoStream.

private DataScanner scanNoStream(ScanStatement statement, StatementEvaluationContext context, Transaction transaction, boolean lockRequired, boolean forWrite) throws StatementExecutionException {
    if (transaction != null) {
        transaction.increaseRefcount();
    }
    try {
        boolean sorted = statement.getComparator() != null;
        boolean sortedByClusteredIndex = statement.getComparator() != null && statement.getComparator().isOnlyPrimaryKeyAndAscending() && keyToPageSortedAscending;
        final Projection projection = statement.getProjection();
        boolean applyProjectionDuringScan = !sorted && projection != null;
        MaterializedRecordSet recordSet;
        if (applyProjectionDuringScan) {
            recordSet = tableSpaceManager.getDbmanager().getRecordSetFactory().createRecordSet(projection.getFieldNames(), projection.getColumns());
        } else {
            recordSet = tableSpaceManager.getDbmanager().getRecordSetFactory().createRecordSet(table.columnNames, table.columns);
        }
        ScanLimits limits = statement.getLimits();
        int maxRows = limits == null ? 0 : limits.computeMaxRows(context);
        int offset = limits == null ? 0 : limits.computeOffset(context);
        boolean sortDone = false;
        if (maxRows > 0) {
            if (sortedByClusteredIndex) {
                // leverage the sorted nature of the clustered primary key index
                AtomicInteger remaining = new AtomicInteger(maxRows);
                if (offset > 0) {
                    remaining.getAndAdd(offset);
                }
                accessTableData(statement, context, new ScanResultOperation() {

                    private boolean inTransactionData;

                    @Override
                    public void beginNewRecordsInTransactionBlock() {
                        inTransactionData = true;
                    }

                    @Override
                    public void accept(Record record, LockHandle lockHandle) throws StatementExecutionException {
                        try {
                            if (applyProjectionDuringScan) {
                                DataAccessor tuple = projection.map(record.getDataAccessor(table), context);
                                recordSet.add(tuple);
                            } else {
                                recordSet.add(record.getDataAccessor(table));
                            }
                            if (!inTransactionData) {
                                // in the same order as the clustered index
                                if (remaining.decrementAndGet() == 0) {
                                    // we want to receive transaction data uncommitted records too
                                    throw new ExitLoop(true);
                                }
                            }
                        } finally {
                            locksManager.releaseLock(lockHandle);
                        }
                    }
                }, transaction, lockRequired, forWrite);
                // we have to sort data any way, because accessTableData will return partially sorted data
                sortDone = transaction == null;
            } else if (sorted) {
                InStreamTupleSorter sorter = new InStreamTupleSorter(offset + maxRows, statement.getComparator());
                accessTableData(statement, context, new ScanResultOperation() {

                    @Override
                    public void accept(Record record, LockHandle lockHandle) throws StatementExecutionException {
                        try {
                            if (applyProjectionDuringScan) {
                                DataAccessor tuple = projection.map(record.getDataAccessor(table), context);
                                sorter.collect(tuple);
                            } else {
                                sorter.collect(record.getDataAccessor(table));
                            }
                        } finally {
                            locksManager.releaseLock(lockHandle);
                        }
                    }
                }, transaction, lockRequired, forWrite);
                sorter.flushToRecordSet(recordSet);
                sortDone = true;
            } else {
                // if no sort is present the limits can be applying during the scan and perform an early exit
                AtomicInteger remaining = new AtomicInteger(maxRows);
                if (offset > 0) {
                    remaining.getAndAdd(offset);
                }
                accessTableData(statement, context, new ScanResultOperation() {

                    @Override
                    public void accept(Record record, LockHandle lockHandle) throws StatementExecutionException {
                        try {
                            if (applyProjectionDuringScan) {
                                DataAccessor tuple = projection.map(record.getDataAccessor(table), context);
                                recordSet.add(tuple);
                            } else {
                                recordSet.add(record.getDataAccessor(table));
                            }
                            if (remaining.decrementAndGet() == 0) {
                                throw new ExitLoop(false);
                            }
                        } finally {
                            locksManager.releaseLock(lockHandle);
                        }
                    }
                }, transaction, lockRequired, forWrite);
            }
        } else {
            accessTableData(statement, context, new ScanResultOperation() {

                @Override
                public void accept(Record record, LockHandle lockHandle) throws StatementExecutionException {
                    try {
                        if (applyProjectionDuringScan) {
                            DataAccessor tuple = projection.map(record.getDataAccessor(table), context);
                            recordSet.add(tuple);
                        } else {
                            recordSet.add(record.getDataAccessor(table));
                        }
                    } finally {
                        locksManager.releaseLock(lockHandle);
                    }
                }
            }, transaction, lockRequired, forWrite);
        }
        recordSet.writeFinished();
        if (!sortDone) {
            recordSet.sort(statement.getComparator());
        }
        recordSet.applyLimits(statement.getLimits(), context);
        if (!applyProjectionDuringScan) {
            recordSet.applyProjection(statement.getProjection(), context);
        }
        return new SimpleDataScanner(transaction, recordSet);
    } finally {
        if (transaction != null) {
            transaction.decreaseRefCount();
        }
    }
}

Example 3 with ScanLimits

use of herddb.model.ScanLimits in project herddb by diennea.

the class JSQLParserPlanner method buildSelectBody.

private PlannerOp buildSelectBody(String defaultTableSpace, int maxRows, SelectBody selectBody, boolean forceScan) throws StatementExecutionException {
    if (selectBody instanceof SetOperationList) {
        SetOperationList list = (SetOperationList) selectBody;
        return buildSetOperationList(defaultTableSpace, maxRows, list, forceScan);
    }
    checkSupported(selectBody instanceof PlainSelect, selectBody.getClass().getName());
    PlainSelect plainSelect = (PlainSelect) selectBody;
    checkSupported(!plainSelect.getMySqlHintStraightJoin());
    checkSupported(!plainSelect.getMySqlSqlCalcFoundRows());
    checkSupported(!plainSelect.getMySqlSqlNoCache());
    checkSupported(plainSelect.getDistinct() == null);
    checkSupported(plainSelect.getFetch() == null);
    checkSupported(plainSelect.getFirst() == null);
    checkSupported(plainSelect.getForUpdateTable() == null);
    checkSupported(plainSelect.getForXmlPath() == null);
    checkSupported(plainSelect.getHaving() == null);
    checkSupported(plainSelect.getIntoTables() == null);
    checkSupported(plainSelect.getOffset() == null);
    checkSupported(plainSelect.getOptimizeFor() == null);
    checkSupported(plainSelect.getOracleHierarchical() == null);
    checkSupported(plainSelect.getOracleHint() == null);
    checkSupported(plainSelect.getSkip() == null);
    checkSupported(plainSelect.getWait() == null);
    checkSupported(plainSelect.getKsqlWindow() == null);
    FromItem fromItem = plainSelect.getFromItem();
    checkSupported(fromItem instanceof net.sf.jsqlparser.schema.Table);
    OpSchema primaryTableSchema = getTableSchema(defaultTableSpace, (net.sf.jsqlparser.schema.Table) fromItem);
    OpSchema[] joinedTables = new OpSchema[0];
    int totalJoinOutputFieldsCount = primaryTableSchema.columns.length;
    if (plainSelect.getJoins() != null) {
        joinedTables = new OpSchema[plainSelect.getJoins().size()];
        int i = 0;
        for (Join join : plainSelect.getJoins()) {
            checkSupported(!join.isApply());
            checkSupported(!join.isCross());
            checkSupported(!join.isFull() || (join.isFull() && join.isOuter()));
            checkSupported(!join.isSemi());
            checkSupported(!join.isStraight());
            checkSupported(!join.isWindowJoin());
            checkSupported(join.getJoinWindow() == null);
            checkSupported(join.getUsingColumns() == null);
            FromItem rightItem = join.getRightItem();
            checkSupported(rightItem instanceof net.sf.jsqlparser.schema.Table);
            OpSchema joinedTable = getTableSchema(defaultTableSpace, (net.sf.jsqlparser.schema.Table) rightItem);
            joinedTables[i++] = joinedTable;
            totalJoinOutputFieldsCount += joinedTable.columns.length;
        }
    }
    int pos = 0;
    String[] joinOutputFieldnames = new String[totalJoinOutputFieldsCount];
    ColumnRef[] joinOutputColumns = new ColumnRef[totalJoinOutputFieldsCount];
    System.arraycopy(primaryTableSchema.columnNames, 0, joinOutputFieldnames, 0, primaryTableSchema.columnNames.length);
    System.arraycopy(primaryTableSchema.columns, 0, joinOutputColumns, 0, primaryTableSchema.columns.length);
    pos += primaryTableSchema.columnNames.length;
    for (OpSchema joinedTable : joinedTables) {
        System.arraycopy(joinedTable.columnNames, 0, joinOutputFieldnames, pos, joinedTable.columnNames.length);
        System.arraycopy(joinedTable.columns, 0, joinOutputColumns, pos, joinedTable.columns.length);
        pos += joinedTable.columnNames.length;
    }
    OpSchema currentSchema = primaryTableSchema;
    // single JOIN only at the moment
    checkSupported(joinedTables.length <= 1);
    if (joinedTables.length > 0) {
        currentSchema = new OpSchema(primaryTableSchema.tableSpace, null, null, joinOutputFieldnames, joinOutputColumns);
    }
    List<SelectItem> selectItems = plainSelect.getSelectItems();
    checkSupported(!selectItems.isEmpty());
    Predicate predicate = null;
    TupleComparator comparator = null;
    ScanLimits limits = null;
    boolean identityProjection = false;
    Projection projection;
    List<SelectExpressionItem> selectedFields = new ArrayList<>(selectItems.size());
    boolean containsAggregatedFunctions = false;
    if (selectItems.size() == 1 && selectItems.get(0) instanceof AllColumns) {
        projection = Projection.IDENTITY(currentSchema.columnNames, ColumnRef.toColumnsArray(currentSchema.columns));
        identityProjection = true;
    } else {
        checkSupported(!selectItems.isEmpty());
        for (SelectItem item : selectItems) {
            if (item instanceof SelectExpressionItem) {
                SelectExpressionItem selectExpressionItem = (SelectExpressionItem) item;
                selectedFields.add(selectExpressionItem);
                if (SQLParserExpressionCompiler.detectAggregatedFunction(selectExpressionItem.getExpression()) != null) {
                    containsAggregatedFunctions = true;
                }
            } else if (item instanceof AllTableColumns) {
                // expand  alias.* to the full list of columns (according to pyhsical schema!)
                AllTableColumns allTablesColumn = (AllTableColumns) item;
                net.sf.jsqlparser.schema.Table table = allTablesColumn.getTable();
                String tableName = fixMySqlBackTicks(table.getName());
                boolean found = false;
                if (primaryTableSchema.isTableOrAlias(tableName)) {
                    for (ColumnRef col : primaryTableSchema.columns) {
                        net.sf.jsqlparser.schema.Column c = new net.sf.jsqlparser.schema.Column(table, col.name);
                        SelectExpressionItem selectExpressionItem = new SelectExpressionItem(c);
                        selectedFields.add(selectExpressionItem);
                        found = true;
                    }
                } else {
                    for (OpSchema joinedTableSchema : joinedTables) {
                        if (joinedTableSchema.isTableOrAlias(tableName)) {
                            for (ColumnRef col : joinedTableSchema.columns) {
                                net.sf.jsqlparser.schema.Column c = new net.sf.jsqlparser.schema.Column(table, col.name);
                                SelectExpressionItem selectExpressionItem = new SelectExpressionItem(c);
                                selectedFields.add(selectExpressionItem);
                            }
                            found = true;
                            break;
                        }
                    }
                }
                if (!found) {
                    checkSupported(false, "Bad table ref " + tableName + ".*");
                }
            } else {
                checkSupported(false);
            }
        }
        if (!containsAggregatedFunctions) {
            // building the projection
            // we have the current physical schema, we can create references by position (as Calcite does)
            // in order to not need accessing columns by name and also making it easier to support
            // ZeroCopyProjections
            projection = buildProjection(selectedFields, true, currentSchema);
        } else {
            // start by full table scan, the AggregateOp operator will create the final projection
            projection = Projection.IDENTITY(currentSchema.columnNames, ColumnRef.toColumnsArray(currentSchema.columns));
            identityProjection = true;
        }
    }
    TableSpaceManager tableSpaceManager = this.manager.getTableSpaceManager(primaryTableSchema.tableSpace);
    AbstractTableManager tableManager = tableSpaceManager.getTableManager(primaryTableSchema.name);
    Table tableImpl = tableManager.getTable();
    CompiledSQLExpression whereExpression = null;
    if (plainSelect.getWhere() != null) {
        whereExpression = SQLParserExpressionCompiler.compileExpression(plainSelect.getWhere(), currentSchema);
        if (joinedTables.length == 0 && whereExpression != null) {
            SQLRecordPredicate sqlWhere = new SQLRecordPredicate(tableImpl, null, whereExpression);
            IndexUtils.discoverIndexOperations(primaryTableSchema.tableSpace, whereExpression, tableImpl, sqlWhere, selectBody, tableSpaceManager);
            predicate = sqlWhere;
        }
    }
    // start with a TableScan + filters
    ScanStatement scan = new ScanStatement(primaryTableSchema.tableSpace, primaryTableSchema.name, Projection.IDENTITY(primaryTableSchema.columnNames, ColumnRef.toColumnsArray(primaryTableSchema.columns)), predicate, comparator, limits);
    scan.setTableDef(tableImpl);
    PlannerOp op = new BindableTableScanOp(scan);
    PlannerOp[] scanJoinedTables = new PlannerOp[joinedTables.length];
    int ji = 0;
    for (OpSchema joinedTable : joinedTables) {
        ScanStatement scanSecondaryTable = new ScanStatement(joinedTable.tableSpace, joinedTable.name, Projection.IDENTITY(joinedTable.columnNames, ColumnRef.toColumnsArray(joinedTable.columns)), null, null, null);
        scan.setTableDef(tableImpl);
        checkSupported(joinedTable.tableSpace.equalsIgnoreCase(primaryTableSchema.tableSpace));
        PlannerOp opSecondaryTable = new BindableTableScanOp(scanSecondaryTable);
        scanJoinedTables[ji++] = opSecondaryTable;
    }
    if (scanJoinedTables.length > 0) {
        // assuming only one JOIN clause
        Join joinClause = plainSelect.getJoins().get(0);
        List<CompiledSQLExpression> joinConditions = new ArrayList<>();
        if (joinClause.isNatural()) {
            // NATURAL join adds a constraint on every column with the same name
            List<CompiledSQLExpression> naturalJoinConstraints = new ArrayList<>();
            int posInRowLeft = 0;
            for (ColumnRef ref : primaryTableSchema.columns) {
                int posInRowRight = primaryTableSchema.columns.length;
                for (ColumnRef ref2 : joinedTables[0].columns) {
                    // assuming only one join
                    if (ref2.name.equalsIgnoreCase(ref.name)) {
                        CompiledSQLExpression equals = new CompiledEqualsExpression(new AccessCurrentRowExpression(posInRowLeft, ref.type), new AccessCurrentRowExpression(posInRowRight, ref2.type));
                        naturalJoinConstraints.add(equals);
                    }
                    posInRowRight++;
                }
                posInRowLeft++;
            }
            CompiledSQLExpression naturalJoin = new CompiledMultiAndExpression(naturalJoinConstraints.toArray(new CompiledSQLExpression[0]));
            joinConditions.add(naturalJoin);
        }
        // handle "ON" clause
        Expression onExpression = joinClause.getOnExpression();
        if (onExpression != null) {
            // TODO: this works for INNER join, but not for LEFT/RIGHT joins
            CompiledSQLExpression onCondition = SQLParserExpressionCompiler.compileExpression(onExpression, currentSchema);
            joinConditions.add(onCondition);
        }
        op = new JoinOp(joinOutputFieldnames, ColumnRef.toColumnsArray(joinOutputColumns), new int[0], op, new int[0], scanJoinedTables[0], // generateNullsOnLeft
        joinClause.isRight() || (joinClause.isFull() && joinClause.isOuter()), // generateNullsOnRight
        joinClause.isLeft() || (joinClause.isFull() && joinClause.isOuter()), // mergeJoin
        false, // "ON" conditions
        joinConditions);
        // handle "WHERE" in case of JOIN
        if (whereExpression != null) {
            op = new FilterOp(op, whereExpression);
        }
    }
    // add aggregations
    if (containsAggregatedFunctions) {
        checkSupported(scanJoinedTables.length == 0);
        op = planAggregate(selectedFields, currentSchema, op, currentSchema, plainSelect.getGroupBy());
        currentSchema = new OpSchema(currentSchema.tableSpace, currentSchema.name, currentSchema.alias, ColumnRef.toColumnsRefsArray(currentSchema.name, op.getOutputSchema()));
    }
    // add order by
    if (plainSelect.getOrderByElements() != null) {
        op = planSort(op, currentSchema, plainSelect.getOrderByElements());
    }
    // add limit
    if (plainSelect.getLimit() != null) {
        checkSupported(scanJoinedTables.length == 0);
        // cannot mix LIMIT and TOP
        checkSupported(plainSelect.getTop() == null);
        Limit limit = plainSelect.getLimit();
        CompiledSQLExpression offset;
        if (limit.getOffset() != null) {
            offset = SQLParserExpressionCompiler.compileExpression(limit.getOffset(), currentSchema);
        } else {
            offset = new ConstantExpression(0, ColumnTypes.NOTNULL_LONG);
        }
        CompiledSQLExpression rowCount = null;
        if (limit.getRowCount() != null) {
            rowCount = SQLParserExpressionCompiler.compileExpression(limit.getRowCount(), currentSchema);
        }
        op = new LimitOp(op, rowCount, offset);
    }
    if (plainSelect.getTop() != null) {
        checkSupported(scanJoinedTables.length == 0);
        // cannot mix LIMIT and TOP
        checkSupported(plainSelect.getLimit() == null);
        Top limit = plainSelect.getTop();
        CompiledSQLExpression rowCount = null;
        if (limit.getExpression() != null) {
            rowCount = SQLParserExpressionCompiler.compileExpression(limit.getExpression(), currentSchema);
        }
        op = new LimitOp(op, rowCount, new ConstantExpression(0, ColumnTypes.NOTNULL_LONG));
    }
    if (!containsAggregatedFunctions && !identityProjection) {
        // add projection
        op = new ProjectOp(projection, op);
    }
    // additional maxrows from JDBC PreparedStatement
    if (maxRows > 0) {
        op = new LimitOp(op, new ConstantExpression(maxRows, ColumnTypes.NOTNULL_LONG), new ConstantExpression(0, ColumnTypes.NOTNULL_LONG)).optimize();
    }
    return op;
}