Examples with ScanLimitsImpl herddb.model.ScanLimitsImpl used on opensource projects

Example 1 with ScanLimitsImpl

use of herddb.model.ScanLimitsImpl in project herddb by diennea.

the class SQLPlanner method buildSelectStatement.

private ExecutionPlan buildSelectStatement(String defaultTableSpace, Select s, boolean scan, int maxRows) throws StatementExecutionException {
    PlainSelect selectBody = (PlainSelect) s.getSelectBody();
    net.sf.jsqlparser.schema.Table fromTable = (net.sf.jsqlparser.schema.Table) selectBody.getFromItem();
    TableRef mainTable = TableRef.buildFrom(fromTable, defaultTableSpace);
    String mainTableAlias = mainTable.tableAlias;
    String tableSpace = mainTable.tableSpace;
    TableSpaceManager tableSpaceManager = manager.getTableSpaceManager(tableSpace);
    if (tableSpaceManager == null) {
        throw new TableSpaceDoesNotExistException("no such tablespace " + tableSpace + " here at " + manager.getNodeId());
    }
    AbstractTableManager tableManager = tableSpaceManager.getTableManager(mainTable.tableName);
    if (tableManager == null) {
        throw new TableDoesNotExistException("no such table " + mainTable.tableName + " in tablespace " + tableSpace);
    }
    // linked hash map retains the order of insertions
    LinkedHashMap<String, JoinSupport> joins = new LinkedHashMap<>();
    boolean joinPresent = false;
    joins.put(mainTable.tableAlias, new JoinSupport(mainTable, tableManager));
    if (selectBody.getJoins() != null) {
        for (Join join : selectBody.getJoins()) {
            joinPresent = true;
            if (join.isLeft() || join.isCross() || join.isRight() || join.isOuter() || join.isSimple()) {
                throw new StatementExecutionException("unsupported JOIN type: " + join);
            }
            net.sf.jsqlparser.schema.Table joinedTable = (net.sf.jsqlparser.schema.Table) join.getRightItem();
            TableRef joinedTableRef = TableRef.buildFrom(joinedTable, defaultTableSpace);
            if (!joinedTableRef.tableSpace.equalsIgnoreCase(mainTable.tableSpace)) {
                throw new TableDoesNotExistException("unsupported cross-tablespace JOIN " + "between" + mainTable.tableSpace + "." + mainTable.tableName + " and " + joinedTableRef.tableSpace + "." + joinedTableRef.tableName);
            }
            AbstractTableManager joinedTableManager = tableSpaceManager.getTableManager(joinedTableRef.tableName);
            if (joinedTableManager == null) {
                throw new TableDoesNotExistException("no such table " + joinedTableRef.tableName + " in tablespace " + tableSpace);
            }
            JoinSupport joinSupport = new JoinSupport(joinedTableRef, joinedTableManager);
            joins.put(joinedTableRef.tableAlias, joinSupport);
        }
    }
    Projection mainTableProjection;
    Table table = tableManager.getTable();
    boolean allColumns = false;
    boolean containsAggregateFunctions = false;
    for (SelectItem c : selectBody.getSelectItems()) {
        if (c instanceof AllColumns) {
            allColumns = true;
            break;
        } else if (c instanceof AllTableColumns) {
            AllTableColumns allTableColumns = (AllTableColumns) c;
            TableRef ref = TableRef.buildFrom(allTableColumns.getTable(), defaultTableSpace);
            if (!joinPresent && ref.tableAlias.equals(mainTable.tableAlias)) {
                // select a.*  FROM table a
                allColumns = true;
            } else {
                // select a.*, b.* FROM table a JOIN table b
                joins.get(ref.tableAlias).allColumns = true;
            }
        } else if (c instanceof SelectExpressionItem) {
            SelectExpressionItem se = (SelectExpressionItem) c;
            if (isAggregateFunction(se.getExpression())) {
                containsAggregateFunctions = true;
            }
            if (!joinPresent) {
                joins.get(mainTable.tableAlias).selectItems.add(c);
            } else {
                ColumnReferencesDiscovery discoverMainTableAlias = discoverMainTableAlias(se.getExpression());
                String mainTableAliasForItem = discoverMainTableAlias.getMainTableAlias();
                if (discoverMainTableAlias.isContainsMixedAliases()) {
                    throw new StatementExecutionException("unsupported single SELECT ITEM with mixed aliases: " + c);
                }
                if (mainTableAliasForItem == null) {
                    mainTableAliasForItem = mainTable.tableAlias;
                }
                joins.get(mainTableAliasForItem).selectItems.add(c);
            }
        } else {
            throw new StatementExecutionException("unsupported SELECT ITEM type: " + c);
        }
    }
    if (allColumns) {
        mainTableProjection = Projection.IDENTITY(table.columnNames, table.columns);
        for (Map.Entry<String, JoinSupport> join : joins.entrySet()) {
            JoinSupport support = join.getValue();
            support.projection = Projection.IDENTITY(support.table.columnNames, support.table.columns);
            support.allColumns = true;
        }
    } else {
        if (!joinPresent) {
            mainTableProjection = new SQLProjection(table, mainTableAlias, selectBody.getSelectItems());
        } else {
            for (JoinSupport support : joins.values()) {
                if (support.allColumns) {
                    support.projection = Projection.IDENTITY(support.table.columnNames, support.table.columns);
                } else {
                    support.projection = new SQLProjection(support.table, support.tableRef.tableAlias, support.selectItems);
                }
            }
            mainTableProjection = joins.get(mainTableAlias).projection;
        }
    }
    if (scan) {
        if (!joinPresent) {
            SQLRecordPredicate where = selectBody.getWhere() != null ? new SQLRecordPredicate(table, mainTableAlias, selectBody.getWhere()) : null;
            if (where != null) {
                discoverIndexOperations(selectBody.getWhere(), table, mainTableAlias, where, tableSpaceManager);
            }
            Aggregator aggregator = null;
            ScanLimitsImpl scanLimits = null;
            if (containsAggregateFunctions || (selectBody.getGroupByColumnReferences() != null && !selectBody.getGroupByColumnReferences().isEmpty())) {
                aggregator = new SQLAggregator(selectBody.getSelectItems(), selectBody.getGroupByColumnReferences(), manager.getRecordSetFactory());
            }
            TupleComparator comparatorOnScan = null;
            TupleComparator comparatorOnPlan = null;
            if (selectBody.getOrderByElements() != null && !selectBody.getOrderByElements().isEmpty()) {
                if (aggregator != null) {
                    comparatorOnPlan = SingleColumnSQLTupleComparator.make(mainTableAlias, selectBody.getOrderByElements(), null);
                } else {
                    comparatorOnScan = SingleColumnSQLTupleComparator.make(mainTableAlias, selectBody.getOrderByElements(), table.primaryKey);
                }
            }
            Limit limit = selectBody.getLimit();
            Top top = selectBody.getTop();
            if (limit != null && top != null) {
                throw new StatementExecutionException("LIMIT and TOP cannot be used on the same query");
            }
            if (limit != null) {
                if (limit.isLimitAll() || limit.isLimitNull() || limit.getOffset() instanceof JdbcParameter) {
                    throw new StatementExecutionException("Invalid LIMIT clause (limit=" + limit + ")");
                }
                if (maxRows > 0 && limit.getRowCount() instanceof JdbcParameter) {
                    throw new StatementExecutionException("Invalid LIMIT clause (limit=" + limit + ") and JDBC setMaxRows=" + maxRows);
                }
                int rowCount;
                int rowCountJdbcParameter = -1;
                if (limit.getRowCount() instanceof JdbcParameter) {
                    rowCount = -1;
                    rowCountJdbcParameter = ((JdbcParameter) limit.getRowCount()).getIndex() - 1;
                } else {
                    rowCount = ((Number) resolveValue(limit.getRowCount(), false)).intValue();
                }
                int offset = limit.getOffset() != null ? ((Number) resolveValue(limit.getOffset(), false)).intValue() : 0;
                scanLimits = new ScanLimitsImpl(rowCount, offset, rowCountJdbcParameter + 1);
            } else if (top != null) {
                if (top.isPercentage() || top.getExpression() == null) {
                    throw new StatementExecutionException("Invalid TOP clause (top=" + top + ")");
                }
                try {
                    int rowCount = Integer.parseInt(resolveValue(top.getExpression(), false) + "");
                    scanLimits = new ScanLimitsImpl(rowCount, 0);
                } catch (NumberFormatException error) {
                    throw new StatementExecutionException("Invalid TOP clause: " + error, error);
                }
            }
            if (maxRows > 0) {
                if (scanLimits == null) {
                    scanLimits = new ScanLimitsImpl(maxRows, 0);
                } else if (scanLimits.getMaxRows() <= 0 || scanLimits.getMaxRows() > maxRows) {
                    scanLimits = new ScanLimitsImpl(maxRows, scanLimits.getOffset());
                }
            }
            ScanLimitsImpl limitsOnScan = null;
            ScanLimitsImpl limitsOnPlan = null;
            if (aggregator != null) {
                limitsOnPlan = scanLimits;
            } else {
                limitsOnScan = scanLimits;
            }
            try {
                ScanStatement statement = new ScanStatement(tableSpace, mainTable.tableName, mainTableProjection, where, comparatorOnScan, limitsOnScan);
                return ExecutionPlan.make(statement, aggregator, limitsOnPlan, comparatorOnPlan);
            } catch (IllegalArgumentException err) {
                throw new StatementExecutionException(err);
            }
        } else {
            if (containsAggregateFunctions || (selectBody.getGroupByColumnReferences() != null && !selectBody.getGroupByColumnReferences().isEmpty())) {
                throw new StatementExecutionException("AGGREGATEs are not yet supported with JOIN");
            }
            Limit limit = selectBody.getLimit();
            Top top = selectBody.getTop();
            if (limit != null && top != null) {
                throw new StatementExecutionException("LIMIT and TOP cannot be used on the same query");
            }
            ScanLimitsImpl scanLimits = null;
            if (limit != null) {
                if (limit.isLimitAll() || limit.isLimitNull() || limit.getOffset() instanceof JdbcParameter) {
                    throw new StatementExecutionException("Invalid LIMIT clause (limit=" + limit + ")");
                }
                if (maxRows > 0 && limit.getRowCount() instanceof JdbcParameter) {
                    throw new StatementExecutionException("Invalid LIMIT clause (limit=" + limit + ") and JDBC setMaxRows=" + maxRows);
                }
                int rowCount;
                int rowCountJdbcParameter = -1;
                if (limit.getRowCount() instanceof JdbcParameter) {
                    rowCount = -1;
                    rowCountJdbcParameter = ((JdbcParameter) limit.getRowCount()).getIndex() - 1;
                } else {
                    rowCount = ((Number) resolveValue(limit.getRowCount(), false)).intValue();
                }
                int offset = limit.getOffset() != null ? ((Number) resolveValue(limit.getOffset(), false)).intValue() : 0;
                scanLimits = new ScanLimitsImpl(rowCount, offset, rowCountJdbcParameter + 1);
            } else if (top != null) {
                if (top.isPercentage() || top.getExpression() == null) {
                    throw new StatementExecutionException("Invalid TOP clause");
                }
                try {
                    int rowCount = Integer.parseInt(resolveValue(top.getExpression(), false) + "");
                    scanLimits = new ScanLimitsImpl(rowCount, 0);
                } catch (NumberFormatException error) {
                    throw new StatementExecutionException("Invalid TOP clause: " + error, error);
                }
            }
            if (maxRows > 0) {
                if (scanLimits == null) {
                    scanLimits = new ScanLimitsImpl(maxRows, 0);
                } else if (scanLimits.getMaxRows() <= 0 || scanLimits.getMaxRows() > maxRows) {
                    scanLimits = new ScanLimitsImpl(maxRows, scanLimits.getOffset());
                }
            }
            List<ColumnReferencesDiscovery> conditionsOnJoinedResult = new ArrayList<>();
            List<ScanStatement> scans = new ArrayList<>();
            for (Map.Entry<String, JoinSupport> join : joins.entrySet()) {
                String alias = join.getKey();
                JoinSupport joinSupport = join.getValue();
                Expression collectedConditionsForAlias = collectConditionsForAlias(alias, selectBody.getWhere(), conditionsOnJoinedResult, mainTableAlias);
                LOG.severe("Collected WHERE for alias " + alias + ": " + collectedConditionsForAlias);
                if (collectedConditionsForAlias == null) {
                    joinSupport.predicate = null;
                } else {
                    joinSupport.predicate = new SQLRecordPredicate(join.getValue().table, alias, collectedConditionsForAlias);
                }
            }
            for (Join join : selectBody.getJoins()) {
                if (join.getOnExpression() != null) {
                    ColumnReferencesDiscovery discoverMainTableAliasForJoinCondition = discoverMainTableAlias(join.getOnExpression());
                    conditionsOnJoinedResult.add(discoverMainTableAliasForJoinCondition);
                    LOG.severe("Collected ON-condition on final JOIN result: " + join.getOnExpression());
                }
            }
            for (ColumnReferencesDiscovery e : conditionsOnJoinedResult) {
                LOG.severe("Collected WHERE on final JOIN result: " + e.getExpression());
                for (Map.Entry<String, List<net.sf.jsqlparser.schema.Column>> entry : e.getColumnsByTable().entrySet()) {
                    String tableAlias = entry.getKey();
                    List<net.sf.jsqlparser.schema.Column> filteredColumnsOnJoin = entry.getValue();
                    LOG.severe("for  TABLE " + tableAlias + " we need to load " + filteredColumnsOnJoin);
                    JoinSupport support = joins.get(tableAlias);
                    if (support == null) {
                        throw new StatementExecutionException("invalid table alias " + tableAlias);
                    }
                    if (!support.allColumns) {
                        for (net.sf.jsqlparser.schema.Column c : filteredColumnsOnJoin) {
                            support.selectItems.add(new SelectExpressionItem(c));
                        }
                        support.projection = new SQLProjection(support.table, support.tableRef.tableAlias, support.selectItems);
                    }
                }
            }
            Map<String, Table> tables = new HashMap<>();
            for (Map.Entry<String, JoinSupport> join : joins.entrySet()) {
                JoinSupport joinSupport = join.getValue();
                tables.put(join.getKey(), joinSupport.table);
                ScanStatement statement = new ScanStatement(tableSpace, joinSupport.table.name, joinSupport.projection, joinSupport.predicate, null, null);
                scans.add(statement);
            }
            TuplePredicate joinFilter = null;
            if (!conditionsOnJoinedResult.isEmpty()) {
                joinFilter = new SQLRecordPredicate(null, null, composeAndExpression(conditionsOnJoinedResult));
            }
            Projection joinProjection = null;
            if (!allColumns) {
                joinProjection = new SQLProjection(tableSpace, tables, selectBody.getSelectItems());
            }
            TupleComparator comparatorOnPlan = null;
            if (selectBody.getOrderByElements() != null && !selectBody.getOrderByElements().isEmpty()) {
                comparatorOnPlan = SingleColumnSQLTupleComparator.make(mainTableAlias, selectBody.getOrderByElements(), null);
            }
            try {
                return ExecutionPlan.joinedScan(scans, joinFilter, joinProjection, scanLimits, comparatorOnPlan);
            } catch (IllegalArgumentException err) {
                throw new StatementExecutionException(err);
            }
        }
    } else {
        if (selectBody.getWhere() == null) {
            throw new StatementExecutionException("unsupported GET without WHERE");
        }
        if (joinPresent) {
            throw new StatementExecutionException("unsupported GET with JOIN");
        }
        // SELECT * FROM WHERE KEY=? AND ....
        SQLRecordKeyFunction keyFunction = findPrimaryKeyIndexSeek(selectBody.getWhere(), table, mainTableAlias);
        if (keyFunction == null || !keyFunction.isFullPrimaryKey()) {
            throw new StatementExecutionException("unsupported GET not on PK, bad where clause: " + selectBody.getWhere() + " (" + selectBody.getWhere().getClass() + ")");
        }
        Predicate where = buildSimplePredicate(selectBody.getWhere(), table, mainTableAlias);
        try {
            return ExecutionPlan.simple(new GetStatement(tableSpace, mainTable.tableName, keyFunction, where, false));
        } catch (IllegalArgumentException err) {
            throw new StatementExecutionException(err);
        }
    }
}

Example 2 with ScanLimitsImpl

use of herddb.model.ScanLimitsImpl in project herddb by diennea.

the class RecordSetSuite method testLimitsAfterEndSwap.

@Test
public void testLimitsAfterEndSwap() throws Exception {
    RecordSetFactory factory = buildRecordSetFactory(1);
    Column[] columns = new Column[2];
    columns[0] = Column.column("s1", ColumnTypes.STRING);
    columns[1] = Column.column("n1", ColumnTypes.LONG);
    String[] fieldNames = Column.buildFieldNamesList(columns);
    try (MaterializedRecordSet rs = factory.createRecordSet(fieldNames, columns)) {
        for (int i = 0; i < 100; i++) {
            Map<String, Object> record = new HashMap<>();
            String s1 = "test_" + i;
            record.put("s1", s1);
            record.put("n1", i);
            rs.add(new Tuple(record, fieldNames));
        }
        rs.writeFinished();
        rs.applyLimits(new ScanLimitsImpl(20, 100000), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT());
        for (DataAccessor t : rs) {
            fail();
        }
    }
}

Example 3 with ScanLimitsImpl

use of herddb.model.ScanLimitsImpl in project herddb by diennea.

the class RecordSetSuite method testLimitsNoSwap.

@Test
public void testLimitsNoSwap() throws Exception {
    RecordSetFactory factory = buildRecordSetFactory(Integer.MAX_VALUE);
    Column[] columns = new Column[2];
    columns[0] = Column.column("s1", ColumnTypes.STRING);
    columns[1] = Column.column("n1", ColumnTypes.LONG);
    String[] fieldNames = Column.buildFieldNamesList(columns);
    try (MaterializedRecordSet rs = factory.createRecordSet(fieldNames, columns)) {
        Set<String> expected_s1 = new HashSet<>();
        Set<Integer> expected_n1 = new HashSet<>();
        for (int i = 0; i < 100; i++) {
            Map<String, Object> record = new HashMap<>();
            String s1 = "test_" + i;
            record.put("s1", s1);
            record.put("n1", i);
            if (i >= 10 && i < 30) {
                expected_s1.add(s1);
                expected_n1.add(i);
            }
            rs.add(new Tuple(record, fieldNames));
        }
        rs.writeFinished();
        rs.applyLimits(new ScanLimitsImpl(20, 10), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT());
        for (DataAccessor t : rs) {
            expected_s1.remove(t.get("s1").toString());
            expected_n1.remove(t.get("n1"));
        }
        assertTrue(expected_n1.isEmpty());
        assertTrue(expected_s1.isEmpty());
    }
}

Example 4 with ScanLimitsImpl

use of herddb.model.ScanLimitsImpl in project herddb by diennea.

the class RecordSetSuite method testLimitsSwap.

@Test
public void testLimitsSwap() throws Exception {
    RecordSetFactory factory = buildRecordSetFactory(1);
    Column[] columns = new Column[2];
    columns[0] = Column.column("s1", ColumnTypes.STRING);
    columns[1] = Column.column("n1", ColumnTypes.LONG);
    String[] fieldNames = Column.buildFieldNamesList(columns);
    try (MaterializedRecordSet rs = factory.createRecordSet(fieldNames, columns)) {
        Set<String> expected_s1 = new HashSet<>();
        Set<Integer> expected_n1 = new HashSet<>();
        for (int i = 0; i < 100; i++) {
            Map<String, Object> record = new HashMap<>();
            String s1 = "test_" + i;
            record.put("s1", s1);
            record.put("n1", i);
            if (i >= 10 && i < 30) {
                expected_s1.add(s1);
                expected_n1.add(i);
            }
            rs.add(new Tuple(record, fieldNames));
        }
        rs.writeFinished();
        rs.applyLimits(new ScanLimitsImpl(20, 10), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT());
        for (DataAccessor t : rs) {
            expected_s1.remove(t.get("s1").toString());
            expected_n1.remove(t.get("n1"));
        }
        assertTrue(expected_n1.isEmpty());
        assertTrue(expected_s1.isEmpty());
    }
}

Example 5 with ScanLimitsImpl

use of herddb.model.ScanLimitsImpl in project herddb by diennea.

the class TableManager method validateAlterTable.

@Override
public void validateAlterTable(Table table, StatementEvaluationContext context) throws StatementExecutionException {
    List<String> columnsChangedFromNullToNotNull = new ArrayList<>();
    for (Column c : this.table.columns) {
        Column newColumnSpecs = table.getColumn(c.name);
        if (newColumnSpecs == null) {
            // dropped column
            LOGGER.log(Level.INFO, "Table {0}.{1} dropping column {2}", new Object[] { table.tablespace, table.name, c.name });
        } else if (newColumnSpecs.type == c.type) {
        // no data type change
        } else if (ColumnTypes.isNotNullToNullConversion(c.type, newColumnSpecs.type)) {
            LOGGER.log(Level.INFO, "Table {0}.{1} making column {2} NULLABLE", new Object[] { table.tablespace, table.name, newColumnSpecs.name });
        } else if (ColumnTypes.isNullToNotNullConversion(c.type, newColumnSpecs.type)) {
            LOGGER.log(Level.INFO, "Table {0}.{1} making column {2} NOT NULL", new Object[] { table.tablespace, table.name, newColumnSpecs.name });
            columnsChangedFromNullToNotNull.add(c.name);
        }
    }
    for (final String column : columnsChangedFromNullToNotNull) {
        LOGGER.log(Level.INFO, "Table {0}.{1} validating column {2}, check for NULL values", new Object[] { table.tablespace, table.name, column });
        ScanStatement scan = new ScanStatement(this.table.tablespace, this.table, new Predicate() {

            @Override
            public boolean evaluate(Record record, StatementEvaluationContext context) throws StatementExecutionException {
                return record.getDataAccessor(table).get(column) == null;
            }
        });
        // fast fail
        scan.setLimits(new ScanLimitsImpl(1, 0));
        boolean foundOneNull = false;
        try (DataScanner scanner = this.scan(scan, context, null, false, false)) {
            foundOneNull = scanner.hasNext();
        } catch (DataScannerException err) {
            throw new StatementExecutionException(err);
        }
        if (foundOneNull) {
            throw new StatementExecutionException("Found a record in table " + table.name + " that contains a NULL value for column " + column + " ALTER command is not possible");
        }
    }
    // if we are adding new FK we have to check that the FK is not violated
    if (table.foreignKeys != null) {
        List<ForeignKeyDef> newForeignKeys;
        if (this.table.foreignKeys == null) {
            newForeignKeys = Arrays.asList(table.foreignKeys);
        } else {
            Set<String> currentKfs = Stream.of(this.table.foreignKeys).map(f -> f.name.toLowerCase()).collect(Collectors.toSet());
            newForeignKeys = Stream.of(table.foreignKeys).filter(fk -> !currentKfs.contains(fk.name)).collect(Collectors.toList());
        }
        for (ForeignKeyDef newFk : newForeignKeys) {
            validateForeignKeyConsistency(newFk, context, null);
        }
    }
}