Examples with LimitOp herddb.model.planner.LimitOp used on opensource projects

Example 1 with LimitOp

use of herddb.model.planner.LimitOp in project herddb by diennea.

the class JSQLParserPlanner method buildSetOperationList.

private PlannerOp buildSetOperationList(String defaultTableSpace, int maxRows, SetOperationList list, boolean forceScan) {
    checkSupported(list.getFetch() == null);
    checkSupported(list.getLimit() == null);
    checkSupported(list.getOffset() == null);
    checkSupported(list.getOrderByElements() == null);
    checkSupported(list.getOperations().size() == 1);
    checkSupported(list.getSelects().size() == 2);
    final SetOperation operation = list.getOperations().get(0);
    checkSupported(operation instanceof UnionOp);
    UnionOp unionOp = (UnionOp) operation;
    // only "UNION ALL"
    checkSupported(unionOp.isAll());
    checkSupported(!unionOp.isDistinct());
    List<PlannerOp> inputs = new ArrayList<>();
    for (SelectBody body : list.getSelects()) {
        inputs.add(buildSelectBody(defaultTableSpace, -1, body, forceScan));
    }
    PlannerOp op = new UnionAllOp(inputs);
    if (maxRows > 0) {
        op = new LimitOp(op, new ConstantExpression(maxRows, ColumnTypes.NOTNULL_LONG), null);
    }
    return op;
}

Example 2 with LimitOp

use of herddb.model.planner.LimitOp in project herddb by diennea.

the class CalcitePlanner method planLimit.

private PlannerOp planLimit(EnumerableLimit op, RelDataType rowType) {
    PlannerOp input = convertRelNode(op.getInput(), rowType, false, false);
    CompiledSQLExpression maxRows = SQLExpressionCompiler.compileExpression(op.fetch);
    CompiledSQLExpression offset = SQLExpressionCompiler.compileExpression(op.offset);
    return new LimitOp(input, maxRows, offset);
}

Example 3 with LimitOp

use of herddb.model.planner.LimitOp in project herddb by diennea.

the class CalcitePlanner method translate.

@Override
public TranslatedQuery translate(String defaultTableSpace, String query, List<Object> parameters, boolean scan, boolean allowCache, boolean returnValues, int maxRows) throws StatementExecutionException {
    ensureDefaultTableSpaceBootedLocally(defaultTableSpace);
    /* Strips out leading comments */
    int idx = SQLUtils.findQueryStart(query);
    if (idx != -1) {
        query = query.substring(idx);
    }
    if (parameters == null) {
        parameters = Collections.emptyList();
    }
    String cacheKey = "scan:" + scan + ",defaultTableSpace:" + defaultTableSpace + ",query:" + query + ",returnValues:" + returnValues + ",maxRows:" + maxRows;
    boolean forceAcquireWriteLock;
    if (// this looks very hacky
    query.endsWith(" FOR UPDATE") && query.substring(0, 6).toLowerCase().equals("select")) {
        forceAcquireWriteLock = true;
        query = query.substring(0, query.length() - " FOR UPDATE".length());
    } else {
        forceAcquireWriteLock = false;
    }
    if (allowCache) {
        ExecutionPlan cached = cache.get(cacheKey);
        if (cached != null) {
            return new TranslatedQuery(cached, new SQLStatementEvaluationContext(query, parameters, forceAcquireWriteLock, false));
        }
    }
    if (isDDL(query)) {
        query = JSQLParserPlanner.rewriteExecuteSyntax(query);
        return fallback.translate(defaultTableSpace, query, parameters, scan, allowCache, returnValues, maxRows);
    }
    if (query.startsWith(TABLE_CONSISTENCY_COMMAND)) {
        query = JSQLParserPlanner.rewriteExecuteSyntax(query);
        return fallback.translate(defaultTableSpace, query, parameters, scan, allowCache, returnValues, maxRows);
    }
    if (query.startsWith(TABLESPACE_CONSISTENCY_COMMAND)) {
        query = JSQLParserPlanner.rewriteExecuteSyntax(query);
        return fallback.translate(defaultTableSpace, query, parameters, scan, allowCache, returnValues, maxRows);
    }
    if (!isCachable(query)) {
        allowCache = false;
    }
    try {
        if (query.startsWith("EXPLAIN ")) {
            query = query.substring("EXPLAIN ".length());
            PlannerResult plan = runPlanner(defaultTableSpace, query);
            boolean upsert = detectUpsert(plan);
            PlannerOp finalPlan = convertRelNode(plan.topNode, plan.originalRowType, returnValues, upsert).optimize();
            ValuesOp values = new ValuesOp(manager.getNodeId(), new String[] { "name", "value" }, new Column[] { column("name", ColumnTypes.STRING), column("value", ColumnTypes.STRING) }, java.util.Arrays.asList(java.util.Arrays.asList(new ConstantExpression("query", ColumnTypes.NOTNULL_STRING), new ConstantExpression(query, ColumnTypes.NOTNULL_STRING)), java.util.Arrays.asList(new ConstantExpression("logicalplan", ColumnTypes.NOTNULL_STRING), new ConstantExpression(RelOptUtil.dumpPlan("", plan.logicalPlan, SqlExplainFormat.TEXT, SqlExplainLevel.ALL_ATTRIBUTES), ColumnTypes.NOTNULL_STRING)), java.util.Arrays.asList(new ConstantExpression("plan", ColumnTypes.NOTNULL_STRING), new ConstantExpression(RelOptUtil.dumpPlan("", plan.topNode, SqlExplainFormat.TEXT, SqlExplainLevel.ALL_ATTRIBUTES), ColumnTypes.NOTNULL_STRING)), java.util.Arrays.asList(new ConstantExpression("finalplan", ColumnTypes.NOTNULL_STRING), new ConstantExpression(finalPlan + "", ColumnTypes.NOTNULL_STRING))));
            ExecutionPlan executionPlan = ExecutionPlan.simple(new SQLPlannedOperationStatement(values), values);
            return new TranslatedQuery(executionPlan, new SQLStatementEvaluationContext(query, parameters, false, false));
        }
        if (query.startsWith("SHOW")) {
            return calculateShowCreateTable(query, defaultTableSpace, parameters, manager);
        }
        PlannerResult plan = runPlanner(defaultTableSpace, query);
        boolean upsert = detectUpsert(plan);
        SQLPlannedOperationStatement sqlPlannedOperationStatement = new SQLPlannedOperationStatement(convertRelNode(plan.topNode, plan.originalRowType, returnValues, upsert).optimize());
        if (LOG.isLoggable(DUMP_QUERY_LEVEL)) {
            LOG.log(DUMP_QUERY_LEVEL, "Query: {0} --HerdDB Plan\n{1}", new Object[] { query, sqlPlannedOperationStatement.getRootOp() });
        }
        if (!scan) {
            ScanStatement scanStatement = sqlPlannedOperationStatement.unwrap(ScanStatement.class);
            if (scanStatement != null) {
                Table tableDef = scanStatement.getTableDef();
                CompiledSQLExpression where = scanStatement.getPredicate().unwrap(CompiledSQLExpression.class);
                SQLRecordKeyFunction keyFunction = IndexUtils.findIndexAccess(where, tableDef.getPrimaryKey(), tableDef, "=", tableDef);
                if (keyFunction == null || !keyFunction.isFullPrimaryKey()) {
                    throw new StatementExecutionException("unsupported GET not on PK, bad where clause: " + query);
                }
                GetStatement get = new GetStatement(scanStatement.getTableSpace(), scanStatement.getTable(), keyFunction, scanStatement.getPredicate(), true);
                ExecutionPlan executionPlan = ExecutionPlan.simple(get);
                if (allowCache) {
                    cache.put(cacheKey, executionPlan);
                }
                return new TranslatedQuery(executionPlan, new SQLStatementEvaluationContext(query, parameters, forceAcquireWriteLock, false));
            }
        }
        if (maxRows > 0) {
            PlannerOp op = new LimitOp(sqlPlannedOperationStatement.getRootOp(), new ConstantExpression(maxRows, ColumnTypes.NOTNULL_LONG), new ConstantExpression(0, ColumnTypes.NOTNULL_LONG)).optimize();
            sqlPlannedOperationStatement = new SQLPlannedOperationStatement(op);
        }
        PlannerOp rootOp = sqlPlannedOperationStatement.getRootOp();
        ExecutionPlan executionPlan;
        if (rootOp.isSimpleStatementWrapper()) {
            executionPlan = ExecutionPlan.simple(rootOp.unwrap(herddb.model.Statement.class), rootOp);
        } else {
            executionPlan = ExecutionPlan.simple(sqlPlannedOperationStatement, rootOp);
        }
        if (allowCache) {
            cache.put(cacheKey, executionPlan);
        }
        return new TranslatedQuery(executionPlan, new SQLStatementEvaluationContext(query, parameters, forceAcquireWriteLock, false));
    } catch (CalciteContextException ex) {
        LOG.log(Level.INFO, "Error while parsing '" + ex.getOriginalStatement() + "'", ex);
        // TODO can this be done better ?
        throw new StatementExecutionException(ex.getMessage());
    } catch (RelConversionException | ValidationException | SqlParseException ex) {
        LOG.log(Level.INFO, "Error while parsing '" + query + "'", ex);
        // TODO can this be done better ?
        throw new StatementExecutionException(ex.getMessage().replace("org.apache.calcite.runtime.CalciteContextException: ", ""), ex);
    } catch (MetadataStorageManagerException ex) {
        LOG.log(Level.INFO, "Error while parsing '" + query + "'", ex);
        throw new StatementExecutionException(ex);
    }
}

Example 4 with LimitOp

use of herddb.model.planner.LimitOp in project herddb by diennea.

the class JSQLParserPlanner method buildSelectBody.

private PlannerOp buildSelectBody(String defaultTableSpace, int maxRows, SelectBody selectBody, boolean forceScan) throws StatementExecutionException {
    if (selectBody instanceof SetOperationList) {
        SetOperationList list = (SetOperationList) selectBody;
        return buildSetOperationList(defaultTableSpace, maxRows, list, forceScan);
    }
    checkSupported(selectBody instanceof PlainSelect, selectBody.getClass().getName());
    PlainSelect plainSelect = (PlainSelect) selectBody;
    checkSupported(!plainSelect.getMySqlHintStraightJoin());
    checkSupported(!plainSelect.getMySqlSqlCalcFoundRows());
    checkSupported(!plainSelect.getMySqlSqlNoCache());
    checkSupported(plainSelect.getDistinct() == null);
    checkSupported(plainSelect.getFetch() == null);
    checkSupported(plainSelect.getFirst() == null);
    checkSupported(plainSelect.getForUpdateTable() == null);
    checkSupported(plainSelect.getForXmlPath() == null);
    checkSupported(plainSelect.getHaving() == null);
    checkSupported(plainSelect.getIntoTables() == null);
    checkSupported(plainSelect.getOffset() == null);
    checkSupported(plainSelect.getOptimizeFor() == null);
    checkSupported(plainSelect.getOracleHierarchical() == null);
    checkSupported(plainSelect.getOracleHint() == null);
    checkSupported(plainSelect.getSkip() == null);
    checkSupported(plainSelect.getWait() == null);
    checkSupported(plainSelect.getKsqlWindow() == null);
    FromItem fromItem = plainSelect.getFromItem();
    checkSupported(fromItem instanceof net.sf.jsqlparser.schema.Table);
    OpSchema primaryTableSchema = getTableSchema(defaultTableSpace, (net.sf.jsqlparser.schema.Table) fromItem);
    OpSchema[] joinedTables = new OpSchema[0];
    int totalJoinOutputFieldsCount = primaryTableSchema.columns.length;
    if (plainSelect.getJoins() != null) {
        joinedTables = new OpSchema[plainSelect.getJoins().size()];
        int i = 0;
        for (Join join : plainSelect.getJoins()) {
            checkSupported(!join.isApply());
            checkSupported(!join.isCross());
            checkSupported(!join.isFull() || (join.isFull() && join.isOuter()));
            checkSupported(!join.isSemi());
            checkSupported(!join.isStraight());
            checkSupported(!join.isWindowJoin());
            checkSupported(join.getJoinWindow() == null);
            checkSupported(join.getUsingColumns() == null);
            FromItem rightItem = join.getRightItem();
            checkSupported(rightItem instanceof net.sf.jsqlparser.schema.Table);
            OpSchema joinedTable = getTableSchema(defaultTableSpace, (net.sf.jsqlparser.schema.Table) rightItem);
            joinedTables[i++] = joinedTable;
            totalJoinOutputFieldsCount += joinedTable.columns.length;
        }
    }
    int pos = 0;
    String[] joinOutputFieldnames = new String[totalJoinOutputFieldsCount];
    ColumnRef[] joinOutputColumns = new ColumnRef[totalJoinOutputFieldsCount];
    System.arraycopy(primaryTableSchema.columnNames, 0, joinOutputFieldnames, 0, primaryTableSchema.columnNames.length);
    System.arraycopy(primaryTableSchema.columns, 0, joinOutputColumns, 0, primaryTableSchema.columns.length);
    pos += primaryTableSchema.columnNames.length;
    for (OpSchema joinedTable : joinedTables) {
        System.arraycopy(joinedTable.columnNames, 0, joinOutputFieldnames, pos, joinedTable.columnNames.length);
        System.arraycopy(joinedTable.columns, 0, joinOutputColumns, pos, joinedTable.columns.length);
        pos += joinedTable.columnNames.length;
    }
    OpSchema currentSchema = primaryTableSchema;
    // single JOIN only at the moment
    checkSupported(joinedTables.length <= 1);
    if (joinedTables.length > 0) {
        currentSchema = new OpSchema(primaryTableSchema.tableSpace, null, null, joinOutputFieldnames, joinOutputColumns);
    }
    List<SelectItem> selectItems = plainSelect.getSelectItems();
    checkSupported(!selectItems.isEmpty());
    Predicate predicate = null;
    TupleComparator comparator = null;
    ScanLimits limits = null;
    boolean identityProjection = false;
    Projection projection;
    List<SelectExpressionItem> selectedFields = new ArrayList<>(selectItems.size());
    boolean containsAggregatedFunctions = false;
    if (selectItems.size() == 1 && selectItems.get(0) instanceof AllColumns) {
        projection = Projection.IDENTITY(currentSchema.columnNames, ColumnRef.toColumnsArray(currentSchema.columns));
        identityProjection = true;
    } else {
        checkSupported(!selectItems.isEmpty());
        for (SelectItem item : selectItems) {
            if (item instanceof SelectExpressionItem) {
                SelectExpressionItem selectExpressionItem = (SelectExpressionItem) item;
                selectedFields.add(selectExpressionItem);
                if (SQLParserExpressionCompiler.detectAggregatedFunction(selectExpressionItem.getExpression()) != null) {
                    containsAggregatedFunctions = true;
                }
            } else if (item instanceof AllTableColumns) {
                // expand  alias.* to the full list of columns (according to pyhsical schema!)
                AllTableColumns allTablesColumn = (AllTableColumns) item;
                net.sf.jsqlparser.schema.Table table = allTablesColumn.getTable();
                String tableName = fixMySqlBackTicks(table.getName());
                boolean found = false;
                if (primaryTableSchema.isTableOrAlias(tableName)) {
                    for (ColumnRef col : primaryTableSchema.columns) {
                        net.sf.jsqlparser.schema.Column c = new net.sf.jsqlparser.schema.Column(table, col.name);
                        SelectExpressionItem selectExpressionItem = new SelectExpressionItem(c);
                        selectedFields.add(selectExpressionItem);
                        found = true;
                    }
                } else {
                    for (OpSchema joinedTableSchema : joinedTables) {
                        if (joinedTableSchema.isTableOrAlias(tableName)) {
                            for (ColumnRef col : joinedTableSchema.columns) {
                                net.sf.jsqlparser.schema.Column c = new net.sf.jsqlparser.schema.Column(table, col.name);
                                SelectExpressionItem selectExpressionItem = new SelectExpressionItem(c);
                                selectedFields.add(selectExpressionItem);
                            }
                            found = true;
                            break;
                        }
                    }
                }
                if (!found) {
                    checkSupported(false, "Bad table ref " + tableName + ".*");
                }
            } else {
                checkSupported(false);
            }
        }
        if (!containsAggregatedFunctions) {
            // building the projection
            // we have the current physical schema, we can create references by position (as Calcite does)
            // in order to not need accessing columns by name and also making it easier to support
            // ZeroCopyProjections
            projection = buildProjection(selectedFields, true, currentSchema);
        } else {
            // start by full table scan, the AggregateOp operator will create the final projection
            projection = Projection.IDENTITY(currentSchema.columnNames, ColumnRef.toColumnsArray(currentSchema.columns));
            identityProjection = true;
        }
    }
    TableSpaceManager tableSpaceManager = this.manager.getTableSpaceManager(primaryTableSchema.tableSpace);
    AbstractTableManager tableManager = tableSpaceManager.getTableManager(primaryTableSchema.name);
    Table tableImpl = tableManager.getTable();
    CompiledSQLExpression whereExpression = null;
    if (plainSelect.getWhere() != null) {
        whereExpression = SQLParserExpressionCompiler.compileExpression(plainSelect.getWhere(), currentSchema);
        if (joinedTables.length == 0 && whereExpression != null) {
            SQLRecordPredicate sqlWhere = new SQLRecordPredicate(tableImpl, null, whereExpression);
            IndexUtils.discoverIndexOperations(primaryTableSchema.tableSpace, whereExpression, tableImpl, sqlWhere, selectBody, tableSpaceManager);
            predicate = sqlWhere;
        }
    }
    // start with a TableScan + filters
    ScanStatement scan = new ScanStatement(primaryTableSchema.tableSpace, primaryTableSchema.name, Projection.IDENTITY(primaryTableSchema.columnNames, ColumnRef.toColumnsArray(primaryTableSchema.columns)), predicate, comparator, limits);
    scan.setTableDef(tableImpl);
    PlannerOp op = new BindableTableScanOp(scan);
    PlannerOp[] scanJoinedTables = new PlannerOp[joinedTables.length];
    int ji = 0;
    for (OpSchema joinedTable : joinedTables) {
        ScanStatement scanSecondaryTable = new ScanStatement(joinedTable.tableSpace, joinedTable.name, Projection.IDENTITY(joinedTable.columnNames, ColumnRef.toColumnsArray(joinedTable.columns)), null, null, null);
        scan.setTableDef(tableImpl);
        checkSupported(joinedTable.tableSpace.equalsIgnoreCase(primaryTableSchema.tableSpace));
        PlannerOp opSecondaryTable = new BindableTableScanOp(scanSecondaryTable);
        scanJoinedTables[ji++] = opSecondaryTable;
    }
    if (scanJoinedTables.length > 0) {
        // assuming only one JOIN clause
        Join joinClause = plainSelect.getJoins().get(0);
        List<CompiledSQLExpression> joinConditions = new ArrayList<>();
        if (joinClause.isNatural()) {
            // NATURAL join adds a constraint on every column with the same name
            List<CompiledSQLExpression> naturalJoinConstraints = new ArrayList<>();
            int posInRowLeft = 0;
            for (ColumnRef ref : primaryTableSchema.columns) {
                int posInRowRight = primaryTableSchema.columns.length;
                for (ColumnRef ref2 : joinedTables[0].columns) {
                    // assuming only one join
                    if (ref2.name.equalsIgnoreCase(ref.name)) {
                        CompiledSQLExpression equals = new CompiledEqualsExpression(new AccessCurrentRowExpression(posInRowLeft, ref.type), new AccessCurrentRowExpression(posInRowRight, ref2.type));
                        naturalJoinConstraints.add(equals);
                    }
                    posInRowRight++;
                }
                posInRowLeft++;
            }
            CompiledSQLExpression naturalJoin = new CompiledMultiAndExpression(naturalJoinConstraints.toArray(new CompiledSQLExpression[0]));
            joinConditions.add(naturalJoin);
        }
        // handle "ON" clause
        Expression onExpression = joinClause.getOnExpression();
        if (onExpression != null) {
            // TODO: this works for INNER join, but not for LEFT/RIGHT joins
            CompiledSQLExpression onCondition = SQLParserExpressionCompiler.compileExpression(onExpression, currentSchema);
            joinConditions.add(onCondition);
        }
        op = new JoinOp(joinOutputFieldnames, ColumnRef.toColumnsArray(joinOutputColumns), new int[0], op, new int[0], scanJoinedTables[0], // generateNullsOnLeft
        joinClause.isRight() || (joinClause.isFull() && joinClause.isOuter()), // generateNullsOnRight
        joinClause.isLeft() || (joinClause.isFull() && joinClause.isOuter()), // mergeJoin
        false, // "ON" conditions
        joinConditions);
        // handle "WHERE" in case of JOIN
        if (whereExpression != null) {
            op = new FilterOp(op, whereExpression);
        }
    }
    // add aggregations
    if (containsAggregatedFunctions) {
        checkSupported(scanJoinedTables.length == 0);
        op = planAggregate(selectedFields, currentSchema, op, currentSchema, plainSelect.getGroupBy());
        currentSchema = new OpSchema(currentSchema.tableSpace, currentSchema.name, currentSchema.alias, ColumnRef.toColumnsRefsArray(currentSchema.name, op.getOutputSchema()));
    }
    // add order by
    if (plainSelect.getOrderByElements() != null) {
        op = planSort(op, currentSchema, plainSelect.getOrderByElements());
    }
    // add limit
    if (plainSelect.getLimit() != null) {
        checkSupported(scanJoinedTables.length == 0);
        // cannot mix LIMIT and TOP
        checkSupported(plainSelect.getTop() == null);
        Limit limit = plainSelect.getLimit();
        CompiledSQLExpression offset;
        if (limit.getOffset() != null) {
            offset = SQLParserExpressionCompiler.compileExpression(limit.getOffset(), currentSchema);
        } else {
            offset = new ConstantExpression(0, ColumnTypes.NOTNULL_LONG);
        }
        CompiledSQLExpression rowCount = null;
        if (limit.getRowCount() != null) {
            rowCount = SQLParserExpressionCompiler.compileExpression(limit.getRowCount(), currentSchema);
        }
        op = new LimitOp(op, rowCount, offset);
    }
    if (plainSelect.getTop() != null) {
        checkSupported(scanJoinedTables.length == 0);
        // cannot mix LIMIT and TOP
        checkSupported(plainSelect.getLimit() == null);
        Top limit = plainSelect.getTop();
        CompiledSQLExpression rowCount = null;
        if (limit.getExpression() != null) {
            rowCount = SQLParserExpressionCompiler.compileExpression(limit.getExpression(), currentSchema);
        }
        op = new LimitOp(op, rowCount, new ConstantExpression(0, ColumnTypes.NOTNULL_LONG));
    }
    if (!containsAggregatedFunctions && !identityProjection) {
        // add projection
        op = new ProjectOp(projection, op);
    }
    // additional maxrows from JDBC PreparedStatement
    if (maxRows > 0) {
        op = new LimitOp(op, new ConstantExpression(maxRows, ColumnTypes.NOTNULL_LONG), new ConstantExpression(0, ColumnTypes.NOTNULL_LONG)).optimize();
    }
    return op;
}