Search in sources :

Example 1 with Predicate

use of herddb.model.Predicate in project herddb by diennea.

the class SQLPlanner method buildSelectStatement.

private ExecutionPlan buildSelectStatement(String defaultTableSpace, Select s, boolean scan, int maxRows) throws StatementExecutionException {
    PlainSelect selectBody = (PlainSelect) s.getSelectBody();
    net.sf.jsqlparser.schema.Table fromTable = (net.sf.jsqlparser.schema.Table) selectBody.getFromItem();
    TableRef mainTable = TableRef.buildFrom(fromTable, defaultTableSpace);
    String mainTableAlias = mainTable.tableAlias;
    String tableSpace = mainTable.tableSpace;
    TableSpaceManager tableSpaceManager = manager.getTableSpaceManager(tableSpace);
    if (tableSpaceManager == null) {
        throw new TableSpaceDoesNotExistException("no such tablespace " + tableSpace + " here at " + manager.getNodeId());
    }
    AbstractTableManager tableManager = tableSpaceManager.getTableManager(mainTable.tableName);
    if (tableManager == null) {
        throw new TableDoesNotExistException("no such table " + mainTable.tableName + " in tablespace " + tableSpace);
    }
    // linked hash map retains the order of insertions
    LinkedHashMap<String, JoinSupport> joins = new LinkedHashMap<>();
    boolean joinPresent = false;
    joins.put(mainTable.tableAlias, new JoinSupport(mainTable, tableManager));
    if (selectBody.getJoins() != null) {
        for (Join join : selectBody.getJoins()) {
            joinPresent = true;
            if (join.isLeft() || join.isCross() || join.isRight() || join.isOuter() || join.isSimple()) {
                throw new StatementExecutionException("unsupported JOIN type: " + join);
            }
            net.sf.jsqlparser.schema.Table joinedTable = (net.sf.jsqlparser.schema.Table) join.getRightItem();
            TableRef joinedTableRef = TableRef.buildFrom(joinedTable, defaultTableSpace);
            if (!joinedTableRef.tableSpace.equalsIgnoreCase(mainTable.tableSpace)) {
                throw new TableDoesNotExistException("unsupported cross-tablespace JOIN " + "between" + mainTable.tableSpace + "." + mainTable.tableName + " and " + joinedTableRef.tableSpace + "." + joinedTableRef.tableName);
            }
            AbstractTableManager joinedTableManager = tableSpaceManager.getTableManager(joinedTableRef.tableName);
            if (joinedTableManager == null) {
                throw new TableDoesNotExistException("no such table " + joinedTableRef.tableName + " in tablespace " + tableSpace);
            }
            JoinSupport joinSupport = new JoinSupport(joinedTableRef, joinedTableManager);
            joins.put(joinedTableRef.tableAlias, joinSupport);
        }
    }
    Projection mainTableProjection;
    Table table = tableManager.getTable();
    boolean allColumns = false;
    boolean containsAggregateFunctions = false;
    for (SelectItem c : selectBody.getSelectItems()) {
        if (c instanceof AllColumns) {
            allColumns = true;
            break;
        } else if (c instanceof AllTableColumns) {
            AllTableColumns allTableColumns = (AllTableColumns) c;
            TableRef ref = TableRef.buildFrom(allTableColumns.getTable(), defaultTableSpace);
            if (!joinPresent && ref.tableAlias.equals(mainTable.tableAlias)) {
                // select a.*  FROM table a
                allColumns = true;
            } else {
                // select a.*, b.* FROM table a JOIN table b
                joins.get(ref.tableAlias).allColumns = true;
            }
        } else if (c instanceof SelectExpressionItem) {
            SelectExpressionItem se = (SelectExpressionItem) c;
            if (isAggregateFunction(se.getExpression())) {
                containsAggregateFunctions = true;
            }
            if (!joinPresent) {
                joins.get(mainTable.tableAlias).selectItems.add(c);
            } else {
                ColumnReferencesDiscovery discoverMainTableAlias = discoverMainTableAlias(se.getExpression());
                String mainTableAliasForItem = discoverMainTableAlias.getMainTableAlias();
                if (discoverMainTableAlias.isContainsMixedAliases()) {
                    throw new StatementExecutionException("unsupported single SELECT ITEM with mixed aliases: " + c);
                }
                if (mainTableAliasForItem == null) {
                    mainTableAliasForItem = mainTable.tableAlias;
                }
                joins.get(mainTableAliasForItem).selectItems.add(c);
            }
        } else {
            throw new StatementExecutionException("unsupported SELECT ITEM type: " + c);
        }
    }
    if (allColumns) {
        mainTableProjection = Projection.IDENTITY(table.columnNames, table.columns);
        for (Map.Entry<String, JoinSupport> join : joins.entrySet()) {
            JoinSupport support = join.getValue();
            support.projection = Projection.IDENTITY(support.table.columnNames, support.table.columns);
            support.allColumns = true;
        }
    } else {
        if (!joinPresent) {
            mainTableProjection = new SQLProjection(table, mainTableAlias, selectBody.getSelectItems());
        } else {
            for (JoinSupport support : joins.values()) {
                if (support.allColumns) {
                    support.projection = Projection.IDENTITY(support.table.columnNames, support.table.columns);
                } else {
                    support.projection = new SQLProjection(support.table, support.tableRef.tableAlias, support.selectItems);
                }
            }
            mainTableProjection = joins.get(mainTableAlias).projection;
        }
    }
    if (scan) {
        if (!joinPresent) {
            SQLRecordPredicate where = selectBody.getWhere() != null ? new SQLRecordPredicate(table, mainTableAlias, selectBody.getWhere()) : null;
            if (where != null) {
                discoverIndexOperations(selectBody.getWhere(), table, mainTableAlias, where, tableSpaceManager);
            }
            Aggregator aggregator = null;
            ScanLimitsImpl scanLimits = null;
            if (containsAggregateFunctions || (selectBody.getGroupByColumnReferences() != null && !selectBody.getGroupByColumnReferences().isEmpty())) {
                aggregator = new SQLAggregator(selectBody.getSelectItems(), selectBody.getGroupByColumnReferences(), manager.getRecordSetFactory());
            }
            TupleComparator comparatorOnScan = null;
            TupleComparator comparatorOnPlan = null;
            if (selectBody.getOrderByElements() != null && !selectBody.getOrderByElements().isEmpty()) {
                if (aggregator != null) {
                    comparatorOnPlan = SingleColumnSQLTupleComparator.make(mainTableAlias, selectBody.getOrderByElements(), null);
                } else {
                    comparatorOnScan = SingleColumnSQLTupleComparator.make(mainTableAlias, selectBody.getOrderByElements(), table.primaryKey);
                }
            }
            Limit limit = selectBody.getLimit();
            Top top = selectBody.getTop();
            if (limit != null && top != null) {
                throw new StatementExecutionException("LIMIT and TOP cannot be used on the same query");
            }
            if (limit != null) {
                if (limit.isLimitAll() || limit.isLimitNull() || limit.getOffset() instanceof JdbcParameter) {
                    throw new StatementExecutionException("Invalid LIMIT clause (limit=" + limit + ")");
                }
                if (maxRows > 0 && limit.getRowCount() instanceof JdbcParameter) {
                    throw new StatementExecutionException("Invalid LIMIT clause (limit=" + limit + ") and JDBC setMaxRows=" + maxRows);
                }
                int rowCount;
                int rowCountJdbcParameter = -1;
                if (limit.getRowCount() instanceof JdbcParameter) {
                    rowCount = -1;
                    rowCountJdbcParameter = ((JdbcParameter) limit.getRowCount()).getIndex() - 1;
                } else {
                    rowCount = ((Number) resolveValue(limit.getRowCount(), false)).intValue();
                }
                int offset = limit.getOffset() != null ? ((Number) resolveValue(limit.getOffset(), false)).intValue() : 0;
                scanLimits = new ScanLimitsImpl(rowCount, offset, rowCountJdbcParameter + 1);
            } else if (top != null) {
                if (top.isPercentage() || top.getExpression() == null) {
                    throw new StatementExecutionException("Invalid TOP clause (top=" + top + ")");
                }
                try {
                    int rowCount = Integer.parseInt(resolveValue(top.getExpression(), false) + "");
                    scanLimits = new ScanLimitsImpl(rowCount, 0);
                } catch (NumberFormatException error) {
                    throw new StatementExecutionException("Invalid TOP clause: " + error, error);
                }
            }
            if (maxRows > 0) {
                if (scanLimits == null) {
                    scanLimits = new ScanLimitsImpl(maxRows, 0);
                } else if (scanLimits.getMaxRows() <= 0 || scanLimits.getMaxRows() > maxRows) {
                    scanLimits = new ScanLimitsImpl(maxRows, scanLimits.getOffset());
                }
            }
            ScanLimitsImpl limitsOnScan = null;
            ScanLimitsImpl limitsOnPlan = null;
            if (aggregator != null) {
                limitsOnPlan = scanLimits;
            } else {
                limitsOnScan = scanLimits;
            }
            try {
                ScanStatement statement = new ScanStatement(tableSpace, mainTable.tableName, mainTableProjection, where, comparatorOnScan, limitsOnScan);
                return ExecutionPlan.make(statement, aggregator, limitsOnPlan, comparatorOnPlan);
            } catch (IllegalArgumentException err) {
                throw new StatementExecutionException(err);
            }
        } else {
            if (containsAggregateFunctions || (selectBody.getGroupByColumnReferences() != null && !selectBody.getGroupByColumnReferences().isEmpty())) {
                throw new StatementExecutionException("AGGREGATEs are not yet supported with JOIN");
            }
            Limit limit = selectBody.getLimit();
            Top top = selectBody.getTop();
            if (limit != null && top != null) {
                throw new StatementExecutionException("LIMIT and TOP cannot be used on the same query");
            }
            ScanLimitsImpl scanLimits = null;
            if (limit != null) {
                if (limit.isLimitAll() || limit.isLimitNull() || limit.getOffset() instanceof JdbcParameter) {
                    throw new StatementExecutionException("Invalid LIMIT clause (limit=" + limit + ")");
                }
                if (maxRows > 0 && limit.getRowCount() instanceof JdbcParameter) {
                    throw new StatementExecutionException("Invalid LIMIT clause (limit=" + limit + ") and JDBC setMaxRows=" + maxRows);
                }
                int rowCount;
                int rowCountJdbcParameter = -1;
                if (limit.getRowCount() instanceof JdbcParameter) {
                    rowCount = -1;
                    rowCountJdbcParameter = ((JdbcParameter) limit.getRowCount()).getIndex() - 1;
                } else {
                    rowCount = ((Number) resolveValue(limit.getRowCount(), false)).intValue();
                }
                int offset = limit.getOffset() != null ? ((Number) resolveValue(limit.getOffset(), false)).intValue() : 0;
                scanLimits = new ScanLimitsImpl(rowCount, offset, rowCountJdbcParameter + 1);
            } else if (top != null) {
                if (top.isPercentage() || top.getExpression() == null) {
                    throw new StatementExecutionException("Invalid TOP clause");
                }
                try {
                    int rowCount = Integer.parseInt(resolveValue(top.getExpression(), false) + "");
                    scanLimits = new ScanLimitsImpl(rowCount, 0);
                } catch (NumberFormatException error) {
                    throw new StatementExecutionException("Invalid TOP clause: " + error, error);
                }
            }
            if (maxRows > 0) {
                if (scanLimits == null) {
                    scanLimits = new ScanLimitsImpl(maxRows, 0);
                } else if (scanLimits.getMaxRows() <= 0 || scanLimits.getMaxRows() > maxRows) {
                    scanLimits = new ScanLimitsImpl(maxRows, scanLimits.getOffset());
                }
            }
            List<ColumnReferencesDiscovery> conditionsOnJoinedResult = new ArrayList<>();
            List<ScanStatement> scans = new ArrayList<>();
            for (Map.Entry<String, JoinSupport> join : joins.entrySet()) {
                String alias = join.getKey();
                JoinSupport joinSupport = join.getValue();
                Expression collectedConditionsForAlias = collectConditionsForAlias(alias, selectBody.getWhere(), conditionsOnJoinedResult, mainTableAlias);
                LOG.severe("Collected WHERE for alias " + alias + ": " + collectedConditionsForAlias);
                if (collectedConditionsForAlias == null) {
                    joinSupport.predicate = null;
                } else {
                    joinSupport.predicate = new SQLRecordPredicate(join.getValue().table, alias, collectedConditionsForAlias);
                }
            }
            for (Join join : selectBody.getJoins()) {
                if (join.getOnExpression() != null) {
                    ColumnReferencesDiscovery discoverMainTableAliasForJoinCondition = discoverMainTableAlias(join.getOnExpression());
                    conditionsOnJoinedResult.add(discoverMainTableAliasForJoinCondition);
                    LOG.severe("Collected ON-condition on final JOIN result: " + join.getOnExpression());
                }
            }
            for (ColumnReferencesDiscovery e : conditionsOnJoinedResult) {
                LOG.severe("Collected WHERE on final JOIN result: " + e.getExpression());
                for (Map.Entry<String, List<net.sf.jsqlparser.schema.Column>> entry : e.getColumnsByTable().entrySet()) {
                    String tableAlias = entry.getKey();
                    List<net.sf.jsqlparser.schema.Column> filteredColumnsOnJoin = entry.getValue();
                    LOG.severe("for  TABLE " + tableAlias + " we need to load " + filteredColumnsOnJoin);
                    JoinSupport support = joins.get(tableAlias);
                    if (support == null) {
                        throw new StatementExecutionException("invalid table alias " + tableAlias);
                    }
                    if (!support.allColumns) {
                        for (net.sf.jsqlparser.schema.Column c : filteredColumnsOnJoin) {
                            support.selectItems.add(new SelectExpressionItem(c));
                        }
                        support.projection = new SQLProjection(support.table, support.tableRef.tableAlias, support.selectItems);
                    }
                }
            }
            Map<String, Table> tables = new HashMap<>();
            for (Map.Entry<String, JoinSupport> join : joins.entrySet()) {
                JoinSupport joinSupport = join.getValue();
                tables.put(join.getKey(), joinSupport.table);
                ScanStatement statement = new ScanStatement(tableSpace, joinSupport.table.name, joinSupport.projection, joinSupport.predicate, null, null);
                scans.add(statement);
            }
            TuplePredicate joinFilter = null;
            if (!conditionsOnJoinedResult.isEmpty()) {
                joinFilter = new SQLRecordPredicate(null, null, composeAndExpression(conditionsOnJoinedResult));
            }
            Projection joinProjection = null;
            if (!allColumns) {
                joinProjection = new SQLProjection(tableSpace, tables, selectBody.getSelectItems());
            }
            TupleComparator comparatorOnPlan = null;
            if (selectBody.getOrderByElements() != null && !selectBody.getOrderByElements().isEmpty()) {
                comparatorOnPlan = SingleColumnSQLTupleComparator.make(mainTableAlias, selectBody.getOrderByElements(), null);
            }
            try {
                return ExecutionPlan.joinedScan(scans, joinFilter, joinProjection, scanLimits, comparatorOnPlan);
            } catch (IllegalArgumentException err) {
                throw new StatementExecutionException(err);
            }
        }
    } else {
        if (selectBody.getWhere() == null) {
            throw new StatementExecutionException("unsupported GET without WHERE");
        }
        if (joinPresent) {
            throw new StatementExecutionException("unsupported GET with JOIN");
        }
        // SELECT * FROM WHERE KEY=? AND ....
        SQLRecordKeyFunction keyFunction = findPrimaryKeyIndexSeek(selectBody.getWhere(), table, mainTableAlias);
        if (keyFunction == null || !keyFunction.isFullPrimaryKey()) {
            throw new StatementExecutionException("unsupported GET not on PK, bad where clause: " + selectBody.getWhere() + " (" + selectBody.getWhere().getClass() + ")");
        }
        Predicate where = buildSimplePredicate(selectBody.getWhere(), table, mainTableAlias);
        try {
            return ExecutionPlan.simple(new GetStatement(tableSpace, mainTable.tableName, keyFunction, where, false));
        } catch (IllegalArgumentException err) {
            throw new StatementExecutionException(err);
        }
    }
}
Also used : TuplePredicate(herddb.model.TuplePredicate) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Projection(herddb.model.Projection) StatementExecutionException(herddb.model.StatementExecutionException) LinkedHashMap(java.util.LinkedHashMap) TuplePredicate(herddb.model.TuplePredicate) Predicate(herddb.model.Predicate) TableSpaceManager(herddb.core.TableSpaceManager) ItemsList(net.sf.jsqlparser.expression.operators.relational.ItemsList) ArrayList(java.util.ArrayList) ExpressionList(net.sf.jsqlparser.expression.operators.relational.ExpressionList) ColumnsList(herddb.model.ColumnsList) MultiExpressionList(net.sf.jsqlparser.expression.operators.relational.MultiExpressionList) List(java.util.List) Top(net.sf.jsqlparser.statement.select.Top) AbstractTableManager(herddb.core.AbstractTableManager) GetStatement(herddb.model.commands.GetStatement) Map(java.util.Map) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) TableSpaceDoesNotExistException(herddb.model.TableSpaceDoesNotExistException) SelectExpressionItem(net.sf.jsqlparser.statement.select.SelectExpressionItem) PlainSelect(net.sf.jsqlparser.statement.select.PlainSelect) TupleComparator(herddb.model.TupleComparator) AllColumns(net.sf.jsqlparser.statement.select.AllColumns) Column(herddb.model.Column) SelectItem(net.sf.jsqlparser.statement.select.SelectItem) ScanStatement(herddb.model.commands.ScanStatement) Table(herddb.model.Table) CreateTable(net.sf.jsqlparser.statement.create.table.CreateTable) JdbcParameter(net.sf.jsqlparser.expression.JdbcParameter) Join(net.sf.jsqlparser.statement.select.Join) Aggregator(herddb.model.Aggregator) ScanLimitsImpl(herddb.model.ScanLimitsImpl) TableDoesNotExistException(herddb.model.TableDoesNotExistException) Expression(net.sf.jsqlparser.expression.Expression) AlterExpression(net.sf.jsqlparser.statement.alter.AlterExpression) BinaryExpression(net.sf.jsqlparser.expression.BinaryExpression) AndExpression(net.sf.jsqlparser.expression.operators.conditional.AndExpression) SignedExpression(net.sf.jsqlparser.expression.SignedExpression) CompiledSQLExpression(herddb.sql.expressions.CompiledSQLExpression) AllTableColumns(net.sf.jsqlparser.statement.select.AllTableColumns) Limit(net.sf.jsqlparser.statement.select.Limit)

Example 2 with Predicate

use of herddb.model.Predicate in project herddb by diennea.

the class CalcitePlanner method planUpdate.

private PlannerOp planUpdate(EnumerableTableModify dml, boolean returnValues) {
    PlannerOp input = convertRelNode(dml.getInput(), null, false);
    List<String> updateColumnList = dml.getUpdateColumnList();
    List<RexNode> sourceExpressionList = dml.getSourceExpressionList();
    final String tableSpace = dml.getTable().getQualifiedName().get(0);
    final String tableName = dml.getTable().getQualifiedName().get(1);
    final TableImpl tableImpl = (TableImpl) dml.getTable().unwrap(org.apache.calcite.schema.Table.class);
    Table table = tableImpl.tableManager.getTable();
    List<CompiledSQLExpression> expressions = new ArrayList<>(sourceExpressionList.size());
    for (RexNode node : sourceExpressionList) {
        CompiledSQLExpression exp = SQLExpressionCompiler.compileExpression(node);
        expressions.add(exp);
    }
    RecordFunction function = new SQLRecordFunction(updateColumnList, table, expressions);
    UpdateStatement update = null;
    if (input instanceof TableScanOp) {
        update = new UpdateStatement(tableSpace, tableName, null, function, null);
    } else if (input instanceof FilterOp) {
        FilterOp filter = (FilterOp) input;
        if (filter.getInput() instanceof TableScanOp) {
            SQLRecordPredicate pred = new SQLRecordPredicate(table, null, filter.getCondition());
            update = new UpdateStatement(tableSpace, tableName, null, function, pred);
        }
    } else if (input instanceof ProjectOp) {
        ProjectOp proj = (ProjectOp) input;
        if (proj.getInput() instanceof TableScanOp) {
            update = new UpdateStatement(tableSpace, tableName, null, function, null);
        } else if (proj.getInput() instanceof FilterOp) {
            FilterOp filter = (FilterOp) proj.getInput();
            if (filter.getInput() instanceof TableScanOp) {
                SQLRecordPredicate pred = new SQLRecordPredicate(table, null, filter.getCondition());
                update = new UpdateStatement(tableSpace, tableName, null, function, pred);
            }
        } else if (proj.getInput() instanceof FilteredTableScanOp) {
            FilteredTableScanOp filter = (FilteredTableScanOp) proj.getInput();
            Predicate pred = filter.getPredicate();
            update = new UpdateStatement(tableSpace, tableName, null, function, pred);
        } else if (proj.getInput() instanceof BindableTableScanOp) {
            BindableTableScanOp filter = (BindableTableScanOp) proj.getInput();
            ScanStatement scan = filter.getStatement();
            if (scan.getComparator() == null && scan.getLimits() == null && scan.getTableDef() != null) {
                Predicate pred = scan.getPredicate();
                update = new UpdateStatement(tableSpace, tableName, null, function, pred);
            }
        }
    }
    if (update != null) {
        return new SimpleUpdateOp(update.setReturnValues(returnValues));
    } else {
        return new UpdateOp(tableSpace, tableName, input, returnValues, function);
    }
}
Also used : SimpleUpdateOp(herddb.model.planner.SimpleUpdateOp) UpdateOp(herddb.model.planner.UpdateOp) FilterOp(herddb.model.planner.FilterOp) ArrayList(java.util.ArrayList) CompiledSQLExpression(herddb.sql.expressions.CompiledSQLExpression) Predicate(herddb.model.Predicate) RecordFunction(herddb.model.RecordFunction) AutoIncrementPrimaryKeyRecordFunction(herddb.model.AutoIncrementPrimaryKeyRecordFunction) ScanStatement(herddb.model.commands.ScanStatement) UpdateStatement(herddb.model.commands.UpdateStatement) PlannerOp(herddb.model.planner.PlannerOp) Table(herddb.model.Table) RelOptTable(org.apache.calcite.plan.RelOptTable) ProjectableFilterableTable(org.apache.calcite.schema.ProjectableFilterableTable) ScannableTable(org.apache.calcite.schema.ScannableTable) AbstractTable(org.apache.calcite.schema.impl.AbstractTable) ModifiableTable(org.apache.calcite.schema.ModifiableTable) FilteredTableScanOp(herddb.model.planner.FilteredTableScanOp) BindableTableScanOp(herddb.model.planner.BindableTableScanOp) TableScanOp(herddb.model.planner.TableScanOp) ProjectOp(herddb.model.planner.ProjectOp) FilteredTableScanOp(herddb.model.planner.FilteredTableScanOp) SimpleUpdateOp(herddb.model.planner.SimpleUpdateOp) RexNode(org.apache.calcite.rex.RexNode) BindableTableScanOp(herddb.model.planner.BindableTableScanOp)

Example 3 with Predicate

use of herddb.model.Predicate in project herddb by diennea.

the class CalcitePlanner method planDelete.

private PlannerOp planDelete(EnumerableTableModify dml) {
    PlannerOp input = convertRelNode(dml.getInput(), null, false);
    final String tableSpace = dml.getTable().getQualifiedName().get(0);
    final String tableName = dml.getTable().getQualifiedName().get(1);
    final TableImpl tableImpl = (TableImpl) dml.getTable().unwrap(org.apache.calcite.schema.Table.class);
    Table table = tableImpl.tableManager.getTable();
    DeleteStatement delete = null;
    if (input instanceof TableScanOp) {
        delete = new DeleteStatement(tableSpace, tableName, null, null);
    } else if (input instanceof FilterOp) {
        FilterOp filter = (FilterOp) input;
        if (filter.getInput() instanceof TableScanOp) {
            SQLRecordPredicate pred = new SQLRecordPredicate(table, null, filter.getCondition());
            delete = new DeleteStatement(tableSpace, tableName, null, pred);
        }
    } else if (input instanceof BindableTableScanOp) {
        BindableTableScanOp filter = (BindableTableScanOp) input;
        Predicate pred = filter.getStatement().getPredicate();
        delete = new DeleteStatement(tableSpace, tableName, null, pred);
    }
    if (delete != null) {
        return new SimpleDeleteOp(delete);
    } else {
        return new DeleteOp(tableSpace, tableName, input);
    }
}
Also used : PlannerOp(herddb.model.planner.PlannerOp) Table(herddb.model.Table) RelOptTable(org.apache.calcite.plan.RelOptTable) ProjectableFilterableTable(org.apache.calcite.schema.ProjectableFilterableTable) ScannableTable(org.apache.calcite.schema.ScannableTable) AbstractTable(org.apache.calcite.schema.impl.AbstractTable) ModifiableTable(org.apache.calcite.schema.ModifiableTable) FilteredTableScanOp(herddb.model.planner.FilteredTableScanOp) BindableTableScanOp(herddb.model.planner.BindableTableScanOp) TableScanOp(herddb.model.planner.TableScanOp) FilterOp(herddb.model.planner.FilterOp) DeleteStatement(herddb.model.commands.DeleteStatement) Predicate(herddb.model.Predicate) SimpleDeleteOp(herddb.model.planner.SimpleDeleteOp) DeleteOp(herddb.model.planner.DeleteOp) SimpleDeleteOp(herddb.model.planner.SimpleDeleteOp) BindableTableScanOp(herddb.model.planner.BindableTableScanOp)

Example 4 with Predicate

use of herddb.model.Predicate in project herddb by diennea.

the class ScanDuringCheckPointTest method bigTableScan.

@Test
public // @Ignore
void bigTableScan() throws Exception {
    int testSize = 10000;
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
        // manager.setMaxLogicalPageSize(10);
        // manager.setMaxPagesUsedMemory(Long.MAX_VALUE);
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        String tableSpaceUUID = manager.getTableSpaceManager("tblspace1").getTableSpaceUUID();
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        for (int i = 0; i < testSize / 2; i++) {
            InsertStatement insert = new InsertStatement(table.tablespace, table.name, RecordSerializer.makeRecord(table, "id", "k" + i, "name", RandomString.getInstance().nextString(50, new StringBuilder().append("testname").append(i).append("_")).toString()));
            assertEquals(1, manager.executeUpdate(insert, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        }
        manager.checkpoint();
        for (int i = testSize / 2; i < testSize; i++) {
            InsertStatement insert = new InsertStatement(table.tablespace, table.name, RecordSerializer.makeRecord(table, "id", "k" + i, "name", RandomString.getInstance().nextString(50, new StringBuilder().append("testname").append(i).append("_")).toString()));
            assertEquals(1, manager.executeUpdate(insert, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        }
        manager.checkpoint();
        assertTrue(manager.getDataStorageManager().getActualNumberOfPages(tableSpaceUUID, table.uuid) > 1);
        TableManagerStats stats = manager.getTableSpaceManager(table.tablespace).getTableManager(table.name).getStats();
        assertEquals(testSize, stats.getTablesize());
        assertTrue(testSize > 100);
        ExecutorService service = Executors.newFixedThreadPool(1);
        Runnable checkPointPerformer = new Runnable() {

            @Override
            public void run() {
                CountDownLatch checkpointDone = new CountDownLatch(1);
                Thread fakeCheckpointThread = new Thread(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            manager.checkpoint();
                        } catch (Throwable t) {
                            t.printStackTrace();
                            fail();
                        }
                        checkpointDone.countDown();
                    }
                });
                fakeCheckpointThread.setDaemon(true);
                try {
                    fakeCheckpointThread.start();
                    checkpointDone.await();
                    fakeCheckpointThread.join();
                } catch (InterruptedException err) {
                    throw new RuntimeException(err);
                }
            }
        };
        AtomicInteger done = new AtomicInteger();
        Predicate slowPredicate = new Predicate() {

            int count = 0;

            @Override
            public boolean evaluate(Record record, StatementEvaluationContext context) throws StatementExecutionException {
                if (count++ % 10 == 0) {
                    // checkpoint will flush buffers, in the middle of the scan
                    try {
                        System.out.println("GO checkpoint !");
                        service.submit(checkPointPerformer).get();
                        done.incrementAndGet();
                    } catch (ExecutionException | InterruptedException err) {
                        throw new StatementExecutionException(err);
                    }
                }
                return true;
            }
        };
        try (DataScanner scan = manager.scan(new ScanStatement(table.tablespace, table, slowPredicate), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)) {
            AtomicInteger count = new AtomicInteger();
            scan.forEach(tuple -> {
                count.incrementAndGet();
            });
            assertEquals(testSize, count.get());
        }
        assertEquals(testSize, stats.getTablesize());
        assertEquals(testSize / 10, done.get());
    }
}
Also used : RandomString(herddb.utils.RandomString) InsertStatement(herddb.model.commands.InsertStatement) StatementExecutionException(herddb.model.StatementExecutionException) Predicate(herddb.model.Predicate) CreateTableSpaceStatement(herddb.model.commands.CreateTableSpaceStatement) DataScanner(herddb.model.DataScanner) TableManagerStats(herddb.core.stats.TableManagerStats) MemoryCommitLogManager(herddb.mem.MemoryCommitLogManager) Record(herddb.model.Record) StatementExecutionException(herddb.model.StatementExecutionException) ExecutionException(java.util.concurrent.ExecutionException) MemoryMetadataStorageManager(herddb.mem.MemoryMetadataStorageManager) ScanStatement(herddb.model.commands.ScanStatement) Table(herddb.model.Table) MemoryDataStorageManager(herddb.mem.MemoryDataStorageManager) CreateTableStatement(herddb.model.commands.CreateTableStatement) CountDownLatch(java.util.concurrent.CountDownLatch) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ExecutorService(java.util.concurrent.ExecutorService) StatementEvaluationContext(herddb.model.StatementEvaluationContext) Test(org.junit.Test)

Example 5 with Predicate

use of herddb.model.Predicate in project herddb by diennea.

the class ScanTest method testTransaction.

@Test
public void testTransaction() throws Exception {
    for (int i = 0; i < 5; i++) {
        Map<String, Object> data = new HashMap<>();
        data.put("id", "key_" + i);
        data.put("number", i);
        Record record = RecordSerializer.toRecord(data, table);
        InsertStatement st = new InsertStatement(tableSpace, tableName, record);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
    }
    BeginTransactionStatement begin = new BeginTransactionStatement(tableSpace);
    long tx = ((TransactionResult) manager.executeStatement(begin, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)).getTransactionId();
    for (int i = 5; i < 10; i++) {
        Map<String, Object> data = new HashMap<>();
        data.put("id", "key_" + i);
        data.put("number", i);
        Record record = RecordSerializer.toRecord(data, table);
        InsertStatement st = new InsertStatement(tableSpace, tableName, record);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx)).getUpdateCount());
    }
    {
        ScanStatement scan = new ScanStatement(tableSpace, table, new Predicate() {

            @Override
            public boolean evaluate(Record record, StatementEvaluationContext context) throws StatementExecutionException {
                return true;
            }
        });
        DataScanner scanner = manager.scan(scan, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        List<?> result = scanner.consume();
        assertEquals(10, result.size());
    }
    for (int i = 0; i < 10; i++) {
        DeleteStatement st = new DeleteStatement(tableSpace, tableName, Bytes.from_string("key_" + i), null);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx)).getUpdateCount());
    }
    {
        ScanStatement scan = new ScanStatement(tableSpace, table, new Predicate() {

            @Override
            public boolean evaluate(Record record, StatementEvaluationContext context) throws StatementExecutionException {
                return true;
            }
        });
        DataScanner scanner = manager.scan(scan, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        List<?> result = scanner.consume();
        assertEquals(0, result.size());
    }
}
Also used : TransactionResult(herddb.model.TransactionResult) HashMap(java.util.HashMap) DeleteStatement(herddb.model.commands.DeleteStatement) InsertStatement(herddb.model.commands.InsertStatement) Predicate(herddb.model.Predicate) DataScanner(herddb.model.DataScanner) TransactionContext(herddb.model.TransactionContext) BeginTransactionStatement(herddb.model.commands.BeginTransactionStatement) Record(herddb.model.Record) List(java.util.List) StatementEvaluationContext(herddb.model.StatementEvaluationContext) ScanStatement(herddb.model.commands.ScanStatement) Test(org.junit.Test)

Aggregations

Predicate (herddb.model.Predicate)14 Record (herddb.model.Record)10 ScanStatement (herddb.model.commands.ScanStatement)8 StatementExecutionException (herddb.model.StatementExecutionException)7 Bytes (herddb.utils.Bytes)7 Table (herddb.model.Table)6 DataScanner (herddb.model.DataScanner)5 LogEntry (herddb.log.LogEntry)4 StatementEvaluationContext (herddb.model.StatementEvaluationContext)4 DeleteStatement (herddb.model.commands.DeleteStatement)4 InsertStatement (herddb.model.commands.InsertStatement)4 DataStorageManagerException (herddb.storage.DataStorageManagerException)4 LockHandle (herddb.utils.LockHandle)4 HashMap (java.util.HashMap)4 List (java.util.List)4 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)4 AtomicLong (java.util.concurrent.atomic.AtomicLong)4 DMLStatementExecutionResult (herddb.model.DMLStatementExecutionResult)3 RecordFunction (herddb.model.RecordFunction)3 UpdateStatement (herddb.model.commands.UpdateStatement)3